@northtek/overstory 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +115 -0
  3. package/dist/build/aggregate.d.ts +9 -0
  4. package/dist/build/aggregate.js +74 -0
  5. package/dist/build/builder.d.ts +52 -0
  6. package/dist/build/builder.js +223 -0
  7. package/dist/build/inmemory.d.ts +16 -0
  8. package/dist/build/inmemory.js +79 -0
  9. package/dist/build/pool.d.ts +3 -0
  10. package/dist/build/pool.js +16 -0
  11. package/dist/build/reflexion.d.ts +15 -0
  12. package/dist/build/reflexion.js +116 -0
  13. package/dist/build/summarize.d.ts +25 -0
  14. package/dist/build/summarize.js +146 -0
  15. package/dist/cli/index.d.ts +2 -0
  16. package/dist/cli/index.js +323 -0
  17. package/dist/core/bm25.d.ts +19 -0
  18. package/dist/core/bm25.js +48 -0
  19. package/dist/core/chunk.d.ts +5 -0
  20. package/dist/core/chunk.js +91 -0
  21. package/dist/core/corpus.d.ts +21 -0
  22. package/dist/core/corpus.js +112 -0
  23. package/dist/core/gate.d.ts +25 -0
  24. package/dist/core/gate.js +155 -0
  25. package/dist/core/hash.d.ts +7 -0
  26. package/dist/core/hash.js +15 -0
  27. package/dist/core/store.d.ts +74 -0
  28. package/dist/core/store.js +66 -0
  29. package/dist/core/types.d.ts +96 -0
  30. package/dist/core/types.js +1 -0
  31. package/dist/fix/prompts.d.ts +4 -0
  32. package/dist/fix/prompts.js +124 -0
  33. package/dist/fix/scan.d.ts +18 -0
  34. package/dist/fix/scan.js +197 -0
  35. package/dist/index.d.ts +28 -0
  36. package/dist/index.js +22 -0
  37. package/dist/llm/anthropic.d.ts +11 -0
  38. package/dist/llm/anthropic.js +46 -0
  39. package/dist/llm/mock.d.ts +5 -0
  40. package/dist/llm/mock.js +20 -0
  41. package/dist/llm/ollama.d.ts +11 -0
  42. package/dist/llm/ollama.js +49 -0
  43. package/dist/llm/provider.d.ts +16 -0
  44. package/dist/llm/provider.js +7 -0
  45. package/dist/llm/resolve.d.ts +14 -0
  46. package/dist/llm/resolve.js +24 -0
  47. package/dist/mcp/server.d.ts +7 -0
  48. package/dist/mcp/server.js +136 -0
  49. package/dist/query/ask.d.ts +53 -0
  50. package/dist/query/ask.js +127 -0
  51. package/dist/query/notarize.d.ts +40 -0
  52. package/dist/query/notarize.js +31 -0
  53. package/dist/registry/github.d.ts +28 -0
  54. package/dist/registry/github.js +113 -0
  55. package/dist/registry/publishClient.d.ts +25 -0
  56. package/dist/registry/publishClient.js +39 -0
  57. package/dist/registry/registry.d.ts +35 -0
  58. package/dist/registry/registry.js +57 -0
  59. package/dist/registry/repoTree.d.ts +9 -0
  60. package/dist/registry/repoTree.js +32 -0
  61. package/dist/serve/app.d.ts +4 -0
  62. package/dist/serve/app.js +809 -0
  63. package/dist/serve/server.d.ts +9 -0
  64. package/dist/serve/server.js +180 -0
  65. package/dist/serve/threads.d.ts +31 -0
  66. package/dist/serve/threads.js +69 -0
  67. package/dist/site/data.d.ts +58 -0
  68. package/dist/site/data.js +88 -0
  69. package/dist/site/generate.d.ts +21 -0
  70. package/dist/site/generate.js +827 -0
  71. package/package.json +64 -0
@@ -0,0 +1,53 @@
1
+ import { Bm25Index } from '../core/bm25.js';
2
+ import type { Claim, CorpusSnapshot, Tree, Verdict } from '../core/types.js';
3
+ import type { ChatProvider } from '../llm/provider.js';
4
+ export interface EvidenceItem {
5
+ claimId: string;
6
+ nodeId: string;
7
+ nodePath: string;
8
+ text: string;
9
+ verdict?: Verdict;
10
+ faithfulness?: Claim['faithfulness'];
11
+ spans: Array<{
12
+ file: string;
13
+ startLine: number;
14
+ endLine: number;
15
+ text: string;
16
+ }>;
17
+ score: number;
18
+ }
19
+ export interface AnswerSentence {
20
+ text: string;
21
+ refs: string[];
22
+ verdict: Verdict;
23
+ }
24
+ export interface AskResult {
25
+ question: string;
26
+ sentences: AnswerSentence[];
27
+ /** Sentences the model produced that cite nothing verifiable — shown flagged, never mixed in. */
28
+ unverifiable: string[];
29
+ /** VERIFIED fraction of answer sentences, 0..1. */
30
+ grounding: number;
31
+ evidence: EvidenceItem[];
32
+ subqueries: string[];
33
+ mode: 'llm' | 'extractive';
34
+ }
35
+ export declare const buildClaimIndex: (tree: Tree) => {
36
+ index: Bm25Index;
37
+ byId: Map<string, {
38
+ claim: Claim;
39
+ nodeId: string;
40
+ }>;
41
+ };
42
+ /** BSHR over the tree: Brainstorm subqueries -> Search claims (BM25) -> Hypothesize an
43
+ * answer grounded in evidence claim ids -> Refine through the gate (every sentence
44
+ * re-verified transitively; unverifiable sentences quarantined, never blended in). */
45
+ export interface AskPhaseEvent {
46
+ phase: 'brainstorm' | 'search' | 'write' | 'gate';
47
+ subqueries?: string[];
48
+ evidenceCount?: number;
49
+ }
50
+ export declare const ask: (question: string, tree: Tree, corpus: CorpusSnapshot, provider: ChatProvider | null, opts?: {
51
+ k?: number;
52
+ onPhase?: (e: AskPhaseEvent) => void;
53
+ }) => Promise<AskResult>;
@@ -0,0 +1,127 @@
1
+ import { z } from 'zod';
2
+ import { Bm25Index } from '../core/bm25.js';
3
+ import { verifyClaim } from '../core/gate.js';
4
+ const subquerySchema = z.object({ subqueries: z.array(z.string().min(2)).min(1).max(4) });
5
+ // refs accepts evidence NUMBERS ([1]-style — reliable for small local models) or raw claim
6
+ // ids (API-grade models); both map back to claim ids before gating.
7
+ const answerSchema = z.object({
8
+ answer: z
9
+ .array(z.object({ text: z.string().min(3), refs: z.array(z.union([z.number().int(), z.string()])).min(1) }))
10
+ .min(1)
11
+ .max(12),
12
+ });
13
+ const ANSWER_SYSTEM = `You answer questions about a codebase using ONLY the verified evidence claims provided. Every sentence must cite the claim ids that support it. If the evidence cannot answer the question, say exactly that in one sentence citing the closest evidence — never fill gaps from general knowledge.`;
14
+ export const buildClaimIndex = (tree) => {
15
+ const byId = new Map();
16
+ const index = new Bm25Index();
17
+ for (const node of Object.values(tree.nodes)) {
18
+ for (const claim of node.claims) {
19
+ byId.set(claim.id, { claim, nodeId: node.id });
20
+ index.add({ id: claim.id, text: `${claim.text} ${node.path} ${node.title}` });
21
+ }
22
+ }
23
+ return { index, byId };
24
+ };
25
+ const evidenceFor = (tree, hits, byId) => hits.map(({ id, score }) => {
26
+ const { claim, nodeId } = byId.get(id);
27
+ return {
28
+ claimId: id,
29
+ nodeId,
30
+ nodePath: tree.nodes[nodeId].path,
31
+ text: claim.text,
32
+ verdict: claim.verdict,
33
+ faithfulness: claim.faithfulness,
34
+ spans: claim.citations
35
+ .filter((c) => c.kind === 'span')
36
+ .map((c) => ({ file: c.span.file, startLine: c.span.startLine, endLine: c.span.endLine, text: c.span.text })),
37
+ score,
38
+ };
39
+ });
40
+ export const ask = async (question, tree, corpus, provider, opts = {}) => {
41
+ const k = opts.k ?? 8;
42
+ const emit = opts.onPhase ?? (() => { });
43
+ const { index, byId } = buildClaimIndex(tree);
44
+ // Brainstorm
45
+ emit({ phase: 'brainstorm' });
46
+ let subqueries = [question];
47
+ if (provider) {
48
+ try {
49
+ const raw = await provider.chat(`Decompose this question about a codebase into 2-4 short search subqueries (keywords, synonyms, likely file/module names). Return ONLY JSON: {"subqueries":["..."]}\n\nQuestion: ${question}`, { json: true });
50
+ const first = raw.indexOf('{');
51
+ subqueries = [question, ...subquerySchema.parse(JSON.parse(first >= 0 ? raw.slice(first, raw.lastIndexOf('}') + 1) : raw)).subqueries];
52
+ }
53
+ catch {
54
+ /* heuristic fallback: the question alone */
55
+ }
56
+ }
57
+ // Search (dedupe, best score wins)
58
+ emit({ phase: 'search', subqueries });
59
+ const merged = new Map();
60
+ for (const q of subqueries) {
61
+ for (const hit of index.search(q, k)) {
62
+ merged.set(hit.id, Math.max(merged.get(hit.id) ?? 0, hit.score));
63
+ }
64
+ }
65
+ const hits = [...merged.entries()]
66
+ .map(([id, score]) => ({ id, score }))
67
+ .sort((a, b) => b.score - a.score)
68
+ .slice(0, k);
69
+ const evidence = evidenceFor(tree, hits, byId);
70
+ // No provider: the verified evidence claims ARE the answer (extractive mode).
71
+ if (!provider) {
72
+ const sentences = evidence.map((e) => ({
73
+ text: e.text,
74
+ refs: [e.claimId],
75
+ verdict: verifyClaim(byId.get(e.claimId).claim, tree, corpus).verdict,
76
+ }));
77
+ const verified = sentences.filter((s) => s.verdict === 'VERIFIED').length;
78
+ return {
79
+ question, sentences, unverifiable: [], evidence, subqueries,
80
+ grounding: sentences.length === 0 ? 0 : verified / sentences.length,
81
+ mode: 'extractive',
82
+ };
83
+ }
84
+ // Hypothesize
85
+ emit({ phase: 'write', evidenceCount: evidence.length });
86
+ const evidenceBlock = evidence.map((e, i) => `[${i + 1}] (${e.nodePath || 'root'}) ${e.text}`).join('\n');
87
+ const sentences = [];
88
+ const unverifiable = [];
89
+ const refToClaimId = (r) => {
90
+ if (typeof r === 'number')
91
+ return evidence[r - 1]?.claimId ?? null;
92
+ const numeric = /^\[?(\d{1,2})\]?$/u.exec(r.trim());
93
+ if (numeric)
94
+ return evidence[Number(numeric[1]) - 1]?.claimId ?? null;
95
+ return byId.has(r) ? r : null;
96
+ };
97
+ try {
98
+ const raw = await provider.chat(`Answer the question using ONLY this numbered evidence. Every answer sentence cites the evidence numbers that support it.\n\nReturn ONLY JSON: {"answer":[{"text":"one sentence","refs":[1,2]}]}\n\nEVIDENCE:\n${evidenceBlock}\n\nQUESTION: ${question}`, { json: true, system: ANSWER_SYSTEM });
99
+ const first = raw.indexOf('{');
100
+ const parsed = answerSchema.parse(JSON.parse(first >= 0 ? raw.slice(first, raw.lastIndexOf('}') + 1) : raw));
101
+ // Refine: every sentence goes through the gate transitively.
102
+ emit({ phase: 'gate' });
103
+ for (const s of parsed.answer) {
104
+ const validRefs = [...new Set(s.refs.map(refToClaimId).filter((r) => r !== null))];
105
+ if (validRefs.length === 0) {
106
+ unverifiable.push(s.text.trim());
107
+ continue;
108
+ }
109
+ const composite = {
110
+ id: 'answer',
111
+ text: s.text,
112
+ citations: validRefs.map((r) => ({ kind: 'claim', ref: { nodeId: byId.get(r).nodeId, claimId: r } })),
113
+ };
114
+ sentences.push({ text: s.text.trim(), refs: validRefs, verdict: verifyClaim(composite, tree, corpus).verdict });
115
+ }
116
+ }
117
+ catch {
118
+ // Provider failed mid-answer: degrade to extractive honestly.
119
+ return { ...(await ask(question, tree, corpus, null, opts)), subqueries, mode: 'extractive' };
120
+ }
121
+ const verified = sentences.filter((s) => s.verdict === 'VERIFIED').length;
122
+ return {
123
+ question, sentences, unverifiable, evidence, subqueries,
124
+ grounding: sentences.length === 0 ? 0 : verified / sentences.length,
125
+ mode: 'llm',
126
+ };
127
+ };
@@ -0,0 +1,40 @@
1
+ import { Bm25Index } from '../core/bm25.js';
2
+ import type { Claim, CorpusSnapshot, Tree, TreeVerification } from '../core/types.js';
3
+ export interface NotarizeInput {
4
+ text: string;
5
+ citations: Array<{
6
+ file: string;
7
+ startLine: number;
8
+ endLine: number;
9
+ }>;
10
+ }
11
+ export interface NotarizedClaim {
12
+ text: string;
13
+ verdict: 'RESOLVED' | 'OUT_OF_CORPUS' | 'UNGROUNDED';
14
+ receipts: Array<{
15
+ file: string;
16
+ lines: string;
17
+ text: string;
18
+ } | {
19
+ missing: string;
20
+ }>;
21
+ corroboration: {
22
+ claimId: string;
23
+ text: string;
24
+ score: number;
25
+ } | null;
26
+ }
27
+ /** Mechanical notarization of externally-drafted claims (host agents, serve API):
28
+ * citations must resolve in the live corpus; receipt text is returned so the caller can
29
+ * self-check support; VERIFIED tree claims corroborate when they match. Deliberately never
30
+ * a claim of semantic truth — verdict names say RESOLVED, not TRUE. */
31
+ export declare const notarizeClaims: (claims: NotarizeInput[], tree: Tree, corpus: CorpusSnapshot, verification: TreeVerification, index: Bm25Index, byId: Map<string, {
32
+ claim: Claim;
33
+ nodeId: string;
34
+ }>) => {
35
+ results: NotarizedClaim[];
36
+ summary: {
37
+ resolved: number;
38
+ of: number;
39
+ };
40
+ };
@@ -0,0 +1,31 @@
1
+ /** Mechanical notarization of externally-drafted claims (host agents, serve API):
2
+ * citations must resolve in the live corpus; receipt text is returned so the caller can
3
+ * self-check support; VERIFIED tree claims corroborate when they match. Deliberately never
4
+ * a claim of semantic truth — verdict names say RESOLVED, not TRUE. */
5
+ export const notarizeClaims = (claims, tree, corpus, verification, index, byId) => {
6
+ const results = claims.map((claim) => {
7
+ if (claim.citations.length === 0)
8
+ return { text: claim.text, verdict: 'UNGROUNDED', receipts: [], corroboration: null };
9
+ const receipts = [];
10
+ let bad = false;
11
+ for (const c of claim.citations) {
12
+ const file = corpus.files.get(c.file.replace(/\\/gu, '/'));
13
+ if (!file || c.startLine > file.lines.length || c.endLine < c.startLine) {
14
+ receipts.push({ missing: `${c.file}:${c.startLine}-${c.endLine}` });
15
+ bad = true;
16
+ continue;
17
+ }
18
+ receipts.push({
19
+ file: c.file,
20
+ lines: `${c.startLine}-${Math.min(c.endLine, file.lines.length)}`,
21
+ text: file.lines.slice(c.startLine - 1, Math.min(c.endLine, file.lines.length)).join('\n').slice(0, 800),
22
+ });
23
+ }
24
+ const top = index.search(claim.text, 1)[0];
25
+ const corroboration = top && verification.verdicts.get(top.id) === 'VERIFIED'
26
+ ? { claimId: top.id, text: byId.get(top.id).claim.text, score: Number(top.score.toFixed(3)) }
27
+ : null;
28
+ return { text: claim.text, verdict: bad ? 'OUT_OF_CORPUS' : 'RESOLVED', receipts, corroboration };
29
+ });
30
+ return { results, summary: { resolved: results.filter((r) => r.verdict === 'RESOLVED').length, of: results.length } };
31
+ };
@@ -0,0 +1,28 @@
1
+ import type { LoadedCorpus } from '../core/corpus.js';
2
+ export interface GithubSnapshot {
3
+ corpus: LoadedCorpus;
4
+ /** Commit sha, recovered from the tarball's root directory name. */
5
+ sha: string;
6
+ ref: string;
7
+ }
8
+ export interface FetchOptions {
9
+ ref?: string;
10
+ maxTarballBytes?: number;
11
+ maxFiles?: number;
12
+ maxFileBytes?: number;
13
+ fetchImpl?: typeof fetch;
14
+ }
15
+ interface TarEntry {
16
+ name: string;
17
+ type: string;
18
+ data: Buffer;
19
+ }
20
+ /** Minimal ustar reader — enough for GitHub codeload tarballs. Handles pax extended
21
+ * headers ('x' path overrides) and skips the pax_global_header ('g'). */
22
+ export declare const readTar: (tar: Buffer) => TarEntry[];
23
+ /** Turn a GitHub codeload tarball into an in-memory corpus. The tarball's root directory
24
+ * is `{repo}-{sha}/` — the sha comes free, no API call needed. */
25
+ export declare const snapshotFromTarball: (tarball: Buffer, ref: string, opts?: FetchOptions) => GithubSnapshot;
26
+ /** Fetch a public GitHub repo snapshot via codeload — no git, no API token, one request. */
27
+ export declare const fetchGithubSnapshot: (owner: string, repo: string, opts?: FetchOptions) => Promise<GithubSnapshot>;
28
+ export {};
@@ -0,0 +1,113 @@
1
+ import { gunzipSync } from 'node:zlib';
2
+ import { normalizeText, sha256, splitLines } from '../core/hash.js';
3
+ const EXCLUDED_DIRS = new Set([
4
+ 'node_modules', '.git', 'dist', 'build', 'out', 'target', 'vendor', 'coverage',
5
+ '.next', '.nuxt', '.overstory', '.cache', '__pycache__', '.venv', 'venv',
6
+ '.idea', '.vscode', '.turbo', '.pytest_cache',
7
+ ]);
8
+ const EXCLUDED_FILENAMES = new Set([
9
+ 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lockb', 'Cargo.lock',
10
+ 'poetry.lock', 'uv.lock', 'Gemfile.lock', 'composer.lock', 'go.sum',
11
+ ]);
12
+ const BINARY_EXTENSIONS = /\.(png|jpe?g|gif|webp|ico|icns|pdf|zip|gz|tgz|bz2|7z|rar|exe|dll|so|dylib|wasm|pyd|class|jar|woff2?|ttf|otf|eot|mp[34]|mov|avi|sqlite|db|bin|pack|idx|lock)$/iu;
13
+ const octal = (buf) => {
14
+ const s = buf.toString('ascii').replace(/\0/gu, '').trim();
15
+ return s.length === 0 ? 0 : parseInt(s, 8);
16
+ };
17
+ /** Minimal ustar reader — enough for GitHub codeload tarballs. Handles pax extended
18
+ * headers ('x' path overrides) and skips the pax_global_header ('g'). */
19
+ export const readTar = (tar) => {
20
+ const entries = [];
21
+ let offset = 0;
22
+ let paxPath = null;
23
+ while (offset + 512 <= tar.length) {
24
+ const header = tar.subarray(offset, offset + 512);
25
+ if (header.every((b) => b === 0))
26
+ break; // end-of-archive blocks
27
+ const rawName = header.subarray(0, 100).toString('utf8').replace(/\0.*$/u, '');
28
+ const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/u, '');
29
+ const size = octal(header.subarray(124, 136));
30
+ const type = header.subarray(156, 157).toString('ascii') || '0';
31
+ const dataStart = offset + 512;
32
+ const data = tar.subarray(dataStart, dataStart + size);
33
+ offset = dataStart + Math.ceil(size / 512) * 512;
34
+ if (type === 'x' || type === 'X') {
35
+ // pax extended header: records "len key=value\n"; capture a path override.
36
+ const text = data.toString('utf8');
37
+ const match = /(?:^|\n)\d+ path=([^\n]+)\n/u.exec(text);
38
+ paxPath = match ? match[1] : null;
39
+ continue;
40
+ }
41
+ if (type === 'g')
42
+ continue; // pax_global_header
43
+ const name = paxPath ?? (prefix ? `${prefix}/${rawName}` : rawName);
44
+ paxPath = null;
45
+ if (type === '0' || type === '')
46
+ entries.push({ name, type, data: Buffer.from(data) });
47
+ }
48
+ return entries;
49
+ };
50
+ const looksBinary = (buf) => buf.subarray(0, 8000).includes(0);
51
+ /** Turn a GitHub codeload tarball into an in-memory corpus. The tarball's root directory
52
+ * is `{repo}-{sha}/` — the sha comes free, no API call needed. */
53
+ export const snapshotFromTarball = (tarball, ref, opts = {}) => {
54
+ const maxFiles = opts.maxFiles ?? 5_000;
55
+ const maxFileBytes = opts.maxFileBytes ?? 1_000_000;
56
+ const entries = readTar(gunzipSync(tarball));
57
+ if (entries.length === 0)
58
+ throw new Error('empty tarball');
59
+ const rootDir = entries[0].name.split('/')[0];
60
+ const shaMatch = /-([0-9a-f]{7,40})$/u.exec(rootDir);
61
+ const sha = shaMatch ? shaMatch[1] : rootDir;
62
+ const files = new Map();
63
+ const skipped = [];
64
+ const sorted = entries
65
+ .map((e) => ({ ...e, rel: e.name.split('/').slice(1).join('/') }))
66
+ .filter((e) => e.rel.length > 0)
67
+ .sort((a, b) => a.rel.localeCompare(b.rel));
68
+ for (const entry of sorted) {
69
+ const rel = entry.rel;
70
+ if (rel.split('/').some((part) => EXCLUDED_DIRS.has(part)))
71
+ continue;
72
+ if (EXCLUDED_FILENAMES.has(rel.split('/').pop() ?? ''))
73
+ continue;
74
+ if (files.size >= maxFiles) {
75
+ skipped.push({ file: rel, reason: 'file-cap' });
76
+ continue;
77
+ }
78
+ if (BINARY_EXTENSIONS.test(rel) || looksBinary(entry.data)) {
79
+ skipped.push({ file: rel, reason: 'binary' });
80
+ continue;
81
+ }
82
+ if (entry.data.length > maxFileBytes) {
83
+ skipped.push({ file: rel, reason: 'too-large' });
84
+ continue;
85
+ }
86
+ const norm = normalizeText(entry.data.toString('utf8'));
87
+ files.set(rel, { hash: sha256(norm), lines: splitLines(norm) });
88
+ }
89
+ return { corpus: { root: `github:${ref}`, files, skipped }, sha, ref };
90
+ };
91
+ /** Fetch a public GitHub repo snapshot via codeload — no git, no API token, one request. */
92
+ export const fetchGithubSnapshot = async (owner, repo, opts = {}) => {
93
+ if (!/^[\w.-]+$/u.test(owner) || !/^[\w.-]+$/u.test(repo))
94
+ throw new Error('invalid owner/repo');
95
+ const ref = opts.ref ?? 'HEAD';
96
+ if (!/^[\w][\w./-]*$/u.test(ref) || ref.includes('..'))
97
+ throw new Error('invalid ref');
98
+ const fetchImpl = opts.fetchImpl ?? fetch;
99
+ const maxTarballBytes = opts.maxTarballBytes ?? 40_000_000;
100
+ const res = await fetchImpl(`https://codeload.github.com/${owner}/${repo}/tar.gz/${ref}`, {
101
+ signal: AbortSignal.timeout(60_000),
102
+ headers: { 'user-agent': 'overstory-registry' },
103
+ });
104
+ if (res.status === 404)
105
+ throw new Error(`GitHub repo not found (or private): ${owner}/${repo}@${ref}`);
106
+ if (!res.ok)
107
+ throw new Error(`GitHub tarball fetch failed: HTTP ${res.status}`);
108
+ const buf = Buffer.from(await res.arrayBuffer());
109
+ if (buf.length > maxTarballBytes) {
110
+ throw new Error(`repo too large for the registry (${Math.round(buf.length / 1e6)}MB tarball; cap ${Math.round(maxTarballBytes / 1e6)}MB)`);
111
+ }
112
+ return snapshotFromTarball(buf, ref, opts);
113
+ };
@@ -0,0 +1,25 @@
1
+ export declare const DEFAULT_REGISTRY: string;
2
+ /** Parse owner/repo out of any common GitHub remote form. */
3
+ export declare const parseGithubRemote: (url: string) => {
4
+ owner: string;
5
+ repo: string;
6
+ } | null;
7
+ export declare const detectGithubRepo: (root: string) => Promise<{
8
+ owner: string;
9
+ repo: string;
10
+ } | null>;
11
+ export interface CheckResponse {
12
+ published: boolean;
13
+ source?: string;
14
+ sha?: string;
15
+ freshness?: number;
16
+ verified?: number;
17
+ claims?: number;
18
+ url?: string;
19
+ badge?: string;
20
+ hint?: string;
21
+ }
22
+ /** Zero-storage publishing: the tree lives in YOUR repo (.overstory/tree.json or a release
23
+ * asset). This asks the registry to fetch your tree from your repo and verify it against
24
+ * your code — the registry stores nothing, ever. */
25
+ export declare const checkPublished: (registry: string, owner: string, repo: string, fetchImpl?: typeof fetch) => Promise<CheckResponse>;
@@ -0,0 +1,39 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { promisify } from 'node:util';
3
+ const execFileP = promisify(execFile);
4
+ export const DEFAULT_REGISTRY = process.env.OVERSTORY_REGISTRY ?? 'https://overstory-virid.vercel.app';
5
+ /** Parse owner/repo out of any common GitHub remote form. */
6
+ export const parseGithubRemote = (url) => {
7
+ const match = /github\.com[/:]([\w.-]+)\/([\w.-]+?)(?:\.git)?\/?$/u.exec(url.trim());
8
+ return match ? { owner: match[1], repo: match[2] } : null;
9
+ };
10
+ export const detectGithubRepo = async (root) => {
11
+ try {
12
+ const { stdout } = await execFileP('git', ['remote', 'get-url', 'origin'], { cwd: root, timeout: 5_000 });
13
+ return parseGithubRemote(stdout);
14
+ }
15
+ catch {
16
+ return null;
17
+ }
18
+ };
19
+ /** Zero-storage publishing: the tree lives in YOUR repo (.overstory/tree.json or a release
20
+ * asset). This asks the registry to fetch your tree from your repo and verify it against
21
+ * your code — the registry stores nothing, ever. */
22
+ export const checkPublished = async (registry, owner, repo, fetchImpl = fetch) => {
23
+ const res = await fetchImpl(`${registry.replace(/\/$/u, '')}/api/check`, {
24
+ method: 'POST',
25
+ signal: AbortSignal.timeout(120_000),
26
+ headers: { 'content-type': 'application/json', 'user-agent': 'overstory-cli' },
27
+ body: JSON.stringify({ owner, repo }),
28
+ });
29
+ let body;
30
+ try {
31
+ body = (await res.json());
32
+ }
33
+ catch {
34
+ throw new Error(`registry returned HTTP ${res.status} with a non-JSON body — check OVERSTORY_REGISTRY points at the registry root`);
35
+ }
36
+ if (!res.ok && res.status !== 404)
37
+ throw new Error(body.error ?? `registry returned HTTP ${res.status}`);
38
+ return body;
39
+ };
@@ -0,0 +1,35 @@
1
+ import type { Tree, TreeVerification } from '../core/types.js';
2
+ import type { GithubSnapshot } from './github.js';
3
+ export interface InstantTreeResult {
4
+ tree: Tree;
5
+ verification: TreeVerification;
6
+ sha: string;
7
+ }
8
+ /** Deterministic, LLM-free tree for any public repo snapshot — the paste-a-URL path. */
9
+ export declare const instantTree: (snapshot: GithubSnapshot, name: string) => Promise<InstantTreeResult>;
10
+ export interface PublishVerdict {
11
+ accepted: boolean;
12
+ freshness: number;
13
+ claims: number;
14
+ verified: number;
15
+ /** Present when rejected: the receipts that failed, capped for response size. */
16
+ failures: Array<{
17
+ claimId: string;
18
+ text: string;
19
+ verdict: string;
20
+ }>;
21
+ }
22
+ /** The registry's core adjudication: an uploaded tree is accepted ONLY if every receipt
23
+ * verifies against the snapshot the server fetched itself from GitHub. Nothing partial is
24
+ * hosted — a 99% tree is a rejection with the failing 1% listed. The uploader is never
25
+ * trusted; the code is. */
26
+ export declare const adjudicatePublish: (rawTree: unknown, snapshot: GithubSnapshot) => {
27
+ verdict: PublishVerdict;
28
+ tree: Tree | null;
29
+ };
30
+ /** Re-verification of a stored tree against a fresh snapshot (freshness decay over time). */
31
+ export declare const reverify: (tree: Tree, snapshot: GithubSnapshot) => {
32
+ freshness: number;
33
+ verified: number;
34
+ claims: number;
35
+ };
@@ -0,0 +1,57 @@
1
+ import { buildTreeFromCorpus } from '../build/inmemory.js';
2
+ import { verifyTree } from '../core/gate.js';
3
+ import { treeSchema } from '../core/store.js';
4
+ /** Deterministic, LLM-free tree for any public repo snapshot — the paste-a-URL path. */
5
+ export const instantTree = async (snapshot, name) => {
6
+ const { tree, verification } = await buildTreeFromCorpus(snapshot.corpus, {
7
+ name,
8
+ provider: null,
9
+ generator: '@northtek/overstory registry (extractive)',
10
+ });
11
+ return { tree, verification, sha: snapshot.sha };
12
+ };
13
+ /** The registry's core adjudication: an uploaded tree is accepted ONLY if every receipt
14
+ * verifies against the snapshot the server fetched itself from GitHub. Nothing partial is
15
+ * hosted — a 99% tree is a rejection with the failing 1% listed. The uploader is never
16
+ * trusted; the code is. */
17
+ export const adjudicatePublish = (rawTree, snapshot) => {
18
+ let tree;
19
+ try {
20
+ tree = treeSchema.parse(rawTree);
21
+ }
22
+ catch {
23
+ return {
24
+ tree: null,
25
+ verdict: { accepted: false, freshness: 0, claims: 0, verified: 0, failures: [{ claimId: '-', text: 'tree failed schema validation', verdict: 'INVALID' }] },
26
+ };
27
+ }
28
+ const verification = verifyTree(tree, snapshot.corpus);
29
+ const failures = [];
30
+ if (verification.freshness < 1) {
31
+ for (const node of Object.values(tree.nodes)) {
32
+ for (const claim of node.claims) {
33
+ const verdict = verification.verdicts.get(claim.id);
34
+ if (verdict && verdict !== 'VERIFIED' && failures.length < 20) {
35
+ failures.push({ claimId: claim.id, text: claim.text.slice(0, 160), verdict });
36
+ }
37
+ }
38
+ }
39
+ }
40
+ const verified = [...verification.verdicts.values()].filter((v) => v === 'VERIFIED').length;
41
+ return {
42
+ tree: verification.freshness === 1 ? tree : null,
43
+ verdict: {
44
+ accepted: verification.freshness === 1,
45
+ freshness: verification.freshness,
46
+ claims: verification.verdicts.size,
47
+ verified,
48
+ failures,
49
+ },
50
+ };
51
+ };
52
+ /** Re-verification of a stored tree against a fresh snapshot (freshness decay over time). */
53
+ export const reverify = (tree, snapshot) => {
54
+ const verification = verifyTree(tree, snapshot.corpus);
55
+ const verified = [...verification.verdicts.values()].filter((v) => v === 'VERIFIED').length;
56
+ return { freshness: verification.freshness, verified, claims: verification.verdicts.size };
57
+ };
@@ -0,0 +1,9 @@
1
+ import type { Tree } from '../core/types.js';
2
+ export interface RepoTreeResult {
3
+ tree: Tree;
4
+ source: string;
5
+ }
6
+ /** Fetch the tree the repo owner committed to their own repository. Null when they haven't
7
+ * published one (the instant extractive tree is the fallback). Schema-invalid trees are
8
+ * treated as absent — never rendered. */
9
+ export declare const fetchRepoTree: (owner: string, repo: string, ref?: string, fetchImpl?: typeof fetch, maxBytes?: number) => Promise<RepoTreeResult | null>;
@@ -0,0 +1,32 @@
1
+ import { treeSchema } from '../core/store.js';
2
+ /** Where a repo's own published tree may live — the OWNER's repo is the database; the
3
+ * registry stores nothing. Checked in order. */
4
+ const candidates = (owner, repo, ref) => [
5
+ `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/.overstory/tree.json`,
6
+ `https://github.com/${owner}/${repo}/releases/latest/download/overstory-tree.json`,
7
+ ];
8
+ /** Fetch the tree the repo owner committed to their own repository. Null when they haven't
9
+ * published one (the instant extractive tree is the fallback). Schema-invalid trees are
10
+ * treated as absent — never rendered. */
11
+ export const fetchRepoTree = async (owner, repo, ref = 'HEAD', fetchImpl = fetch, maxBytes = 30_000_000) => {
12
+ for (const url of candidates(owner, repo, ref)) {
13
+ try {
14
+ const res = await fetchImpl(url, {
15
+ signal: AbortSignal.timeout(30_000),
16
+ headers: { 'user-agent': 'overstory-registry' },
17
+ redirect: 'follow',
18
+ });
19
+ if (!res.ok)
20
+ continue;
21
+ const text = await res.text();
22
+ if (text.length > maxBytes)
23
+ continue;
24
+ const tree = treeSchema.parse(JSON.parse(text));
25
+ return { tree, source: url.includes('raw.githubusercontent') ? '.overstory/tree.json' : 'release asset' };
26
+ }
27
+ catch {
28
+ continue;
29
+ }
30
+ }
31
+ return null;
32
+ };
@@ -0,0 +1,4 @@
1
+ /** The served application shell. All data is client-fetched from /api/*; this template has
2
+ * zero server-side interpolation. Client JS avoids backticks entirely (template-literal
3
+ * safety inside this TS string). */
4
+ export declare const appHtml: () => string;