@zenera/rag 1.1.6 → 1.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +28 -2
  2. package/dist/command.js +41 -727
  3. package/dist/common/embedder.d.ts +3 -0
  4. package/dist/common/embedder.js +64 -0
  5. package/dist/common/locate.d.ts +18 -0
  6. package/dist/{schema → common}/locate.js +42 -28
  7. package/dist/common/manifest.d.ts +50 -0
  8. package/dist/common/manifest.js +62 -0
  9. package/dist/common/progress.d.ts +57 -0
  10. package/dist/common/progress.js +155 -0
  11. package/dist/common/prose.d.ts +13 -0
  12. package/dist/common/prose.js +56 -0
  13. package/dist/index.d.ts +6 -4
  14. package/dist/index.js +6 -4
  15. package/dist/schema/build.js +6 -2
  16. package/dist/schema/command.d.ts +3 -0
  17. package/dist/schema/command.js +819 -0
  18. package/dist/schema/files.d.ts +5 -27
  19. package/dist/schema/files.js +9 -29
  20. package/dist/schema/lookup.d.ts +1 -1
  21. package/dist/schema/lookup.js +1 -1
  22. package/dist/{present.d.ts → schema/present.d.ts} +5 -5
  23. package/dist/{present.js → schema/present.js} +2 -2
  24. package/dist/{query.d.ts → schema/query.d.ts} +1 -1
  25. package/dist/schema/readme.d.ts +6 -0
  26. package/dist/schema/readme.js +122 -0
  27. package/dist/{repl.d.ts → schema/repl.d.ts} +1 -1
  28. package/dist/schema/search.js +2 -1
  29. package/dist/schema/tools.d.ts +1 -1
  30. package/dist/schema/tools.js +97 -6
  31. package/dist/schema/trace.d.ts +52 -0
  32. package/dist/schema/trace.js +144 -0
  33. package/package.json +3 -3
  34. package/dist/schema/locate.d.ts +0 -20
  35. package/dist/schema/progress.d.ts +0 -26
  36. package/dist/schema/progress.js +0 -316
  37. /package/dist/{schema → common}/match.d.ts +0 -0
  38. /package/dist/{schema → common}/match.js +0 -0
  39. /package/dist/{query.js → schema/query.js} +0 -0
  40. /package/dist/{repl.js → schema/repl.js} +0 -0
@@ -0,0 +1,3 @@
1
+ import { type Embedder } from '@zenera/neo';
2
+ export declare function resolveEmbedder(ref: string | undefined): Promise<Embedder>;
3
+ //# sourceMappingURL=embedder.d.ts.map
@@ -0,0 +1,64 @@
1
+ import { bold, CliError, CURATED, cyan, dim, ensureHome, envNames, form, KeyStore, note, PROVIDERS, table, usageError, } from '@zenera/cli/lib';
2
+ import { createEmbedder } from '@zenera/neo';
3
+ // ---------------------------------------------------------------------------
4
+ // Getting an embedder, and saying what the choices are when there is none
5
+ //
6
+ // The keyring is materialised here, and only here. The `zen` frame does not do
7
+ // it, and a command that forgets to looks exactly like a machine with no key:
8
+ // `zen key ls` shows the credential live and the command still says the
9
+ // environment variable is not set. Real env always wins, so a shell that names
10
+ // a key is never overridden by one on disk.
11
+ //
12
+ // Nothing in here is about a particular index. Which model made the vectors is
13
+ // recorded in the manifest and enforced on every later search, so this only has
14
+ // to turn a reference into an embedder — or, given none, into a list worth
15
+ // choosing from.
16
+ // ---------------------------------------------------------------------------
17
+ export async function resolveEmbedder(ref) {
18
+ ensureHome();
19
+ const keys = await KeyStore.open();
20
+ // Asked before materialising, because materialising is what erases the
21
+ // difference between "the environment had it" and "the keyring supplied it".
22
+ const fromEnv = new Set(PROVIDERS.filter((p) => envNames(p).some((n) => process.env[n])));
23
+ keys.materialize();
24
+ if (!ref) {
25
+ throw choices(keys, fromEnv);
26
+ }
27
+ return createEmbedder(ref);
28
+ }
29
+ /**
30
+ * Well-known embedding models per provider, read off the CLI's catalog table so
31
+ * there is one list rather than two that drift. Any ref the registry can parse
32
+ * works; these are the ones worth typing. Anthropic has none because it
33
+ * publishes no embeddings API at all.
34
+ */
35
+ const embeddingsOf = (provider) => CURATED[provider].filter((m) => m.roles.includes('embedding')).map((m) => m.id);
36
+ /** What could be passed, with the ones this machine can actually use first. */
37
+ function choices(keys, fromEnv) {
38
+ const rows = [];
39
+ const rest = [];
40
+ for (const provider of PROVIDERS) {
41
+ for (const model of embeddingsOf(provider)) {
42
+ const source = fromEnv.has(provider)
43
+ ? 'environment'
44
+ : keys.active(provider)
45
+ ? 'keyring'
46
+ : '';
47
+ const row = [` ${cyan(`${provider}:${model}`)}`, dim(source || form(provider).env)];
48
+ (source ? rows : rest).push(row);
49
+ }
50
+ }
51
+ note(bold('Embeddings'));
52
+ for (const line of table([...rows, ...rest])) {
53
+ note(line);
54
+ }
55
+ note('');
56
+ if (rows.length === 0) {
57
+ note(dim(' no provider on this machine has a credential — try: zen key add openai'));
58
+ note('');
59
+ }
60
+ // `pick` is the one that ends the question rather than restating it: it
61
+ // tries them and prints the first that answers.
62
+ return usageError('no embedder named', 'pass --embedding <ref>, or run: zen models pick --embedding');
63
+ }
64
+ //# sourceMappingURL=embedder.js.map
@@ -0,0 +1,18 @@
1
+ import { type IndexKind, type IndexSpec } from './manifest.ts';
2
+ /** How the directory was arrived at, which is what decides whether to say so. */
3
+ export type DirSource = 'flag' | 'env' | 'found' | 'default';
4
+ export interface Located {
5
+ dir: string;
6
+ from: DirSource;
7
+ }
8
+ export interface LocateOptions {
9
+ env?: NodeJS.ProcessEnv;
10
+ /** do not climb above this; the home directory, or the root, by default */
11
+ ceiling?: string;
12
+ }
13
+ export declare function locateIndex(cwd: string, flag: string | undefined, spec: IndexSpec, options?: LocateOptions): Located;
14
+ /** Where a new index goes: the same environment variable, minus the search. */
15
+ export declare function outputDir(cwd: string, flag: string | undefined, spec: IndexSpec, env?: NodeJS.ProcessEnv): string;
16
+ /** An index of this kind is a directory with a manifest saying so; nothing else is asserted. */
17
+ export declare function isIndex(dir: string, kind: IndexKind): boolean;
18
+ //# sourceMappingURL=locate.d.ts.map
@@ -1,22 +1,28 @@
1
1
  import { CliError, EXIT } from '@zenera/cli/lib';
2
- import { readdirSync, statSync } from 'node:fs';
2
+ import { readdirSync, readFileSync } from 'node:fs';
3
3
  import { homedir } from 'node:os';
4
4
  import { dirname, isAbsolute, join, parse, relative, resolve } from 'node:path';
5
- import { MANIFEST_FILE } from "./files.js";
5
+ import { MANIFEST_FILE } from "./manifest.js";
6
6
  // ---------------------------------------------------------------------------
7
7
  // Which index, when nobody said
8
8
  //
9
- // `--dir` and `$ZEN_SCHEMA_DB` are taken as written, missing or not: naming a
10
- // directory that turns out not to hold an index has to fail saying so, because
11
- // quietly using a different one would be a worse answer than an error.
9
+ // `--dir` and the environment variable are taken as written, missing or not:
10
+ // naming a directory that turns out not to hold an index has to fail saying so,
11
+ // because quietly using a different one would be a worse answer than an error.
12
12
  //
13
13
  // With neither, the directory is looked for. There is no list of blessed names
14
14
  // here and there should not be — an index is self-describing, so what is being
15
- // looked for is a `manifest.json`, not a directory called `schema-db`.
16
- // `schema-db` is only the name a *new* index is given, and nothing reads it
17
- // back. The search is nearest-first: this directory, then what is under it,
18
- // then up a level and again, so `/assets/…/whatever` is reachable from
19
- // `/workspace` because the two meet at a shared root on the way up.
15
+ // looked for is a `manifest.json`, not a directory called `schema-db`. That
16
+ // name is only what a *new* index is given, and nothing reads it back. The
17
+ // search is nearest-first: this directory, then what is under it, then up a
18
+ // level and again, so `/assets/…/whatever` is reachable from `/workspace`
19
+ // because the two meet at a shared root on the way up.
20
+ //
21
+ // The manifest is read rather than merely counted, because a tree holding both
22
+ // an API index and a document index has two answers to "the nearest index" and
23
+ // only one of them is the one being asked for. Scoping the search by kind is
24
+ // what stops the other from being found — and stops two subjects that happen to
25
+ // sit side by side from reading as an ambiguity.
20
26
  //
21
27
  // Three things bound it, and each is bounding a different kind of mistake.
22
28
  // Depth and a visit budget bound the cost. The ceiling — the home directory,
@@ -24,11 +30,8 @@ import { MANIFEST_FILE } from "./files.js";
24
30
  // blast radius, because an index in someone else's tree is not yours. And two
25
31
  // indexes at the same distance is an ambiguity rather than a tie to break:
26
32
  // choosing one silently is the one failure worth ruling out entirely, since
27
- // the wrong index does not error, it answers confidently about another API.
33
+ // the wrong index does not error, it answers confidently about another corpus.
28
34
  // ---------------------------------------------------------------------------
29
- /** The name a new index is given. Nothing searches for it; only `index --out` writes it. */
30
- export const DEFAULT_DIR = './schema-db';
31
- export const DIR_ENV = 'ZEN_SCHEMA_DB';
32
35
  /** Far enough to climb out of a package into its workspace, not far enough to roam. */
33
36
  const MAX_LEVELS = 6;
34
37
  /** How far below a directory an index may sit and still count as being in it. */
@@ -37,45 +40,56 @@ const MAX_DEPTH = 3;
37
40
  const MAX_ENTRIES = 128;
38
41
  /** Directories the whole search may read, however it is shaped. */
39
42
  const MAX_VISITS = 400;
40
- export function locateIndex(cwd, flag, options = {}) {
43
+ export function locateIndex(cwd, flag, spec, options = {}) {
41
44
  const env = options.env ?? process.env;
42
45
  if (flag) {
43
46
  return { dir: resolve(cwd, flag), from: 'flag' };
44
47
  }
45
- const named = env[DIR_ENV]?.trim();
48
+ const named = env[spec.envName]?.trim();
46
49
  if (named) {
47
50
  return { dir: resolve(cwd, named), from: 'env' };
48
51
  }
49
- const found = search(resolve(cwd), options.ceiling ?? ceilingFor(cwd));
52
+ const found = search(resolve(cwd), options.ceiling ?? ceilingFor(cwd), spec);
50
53
  // Nothing found still answers with the default, so the error names the
51
54
  // directory everyone expects rather than the last place that was searched.
52
55
  return found
53
56
  ? { dir: found, from: 'found' }
54
- : { dir: resolve(cwd, DEFAULT_DIR), from: 'default' };
57
+ : { dir: resolve(cwd, spec.defaultDir), from: 'default' };
55
58
  }
56
59
  /** Where a new index goes: the same environment variable, minus the search. */
57
- export function outputDir(cwd, flag, env = process.env) {
58
- return resolve(cwd, flag ?? env[DIR_ENV]?.trim() ?? DEFAULT_DIR);
60
+ export function outputDir(cwd, flag, spec, env = process.env) {
61
+ return resolve(cwd, flag ?? env[spec.envName]?.trim() ?? spec.defaultDir);
59
62
  }
60
- /** An index is a directory with a manifest in it; nothing else is asserted here. */
61
- export function isIndex(dir) {
63
+ /** An index of this kind is a directory with a manifest saying so; nothing else is asserted. */
64
+ export function isIndex(dir, kind) {
65
+ let text;
62
66
  try {
63
- return statSync(join(dir, MANIFEST_FILE)).isFile();
67
+ text = readFileSync(join(dir, MANIFEST_FILE), 'utf8');
64
68
  }
65
69
  catch {
66
70
  return false;
67
71
  }
72
+ let found;
73
+ try {
74
+ found = JSON.parse(text).kind;
75
+ }
76
+ catch {
77
+ // A manifest too broken to parse is still an index, and saying so is
78
+ // what gets the caller that error instead of "nothing found".
79
+ return true;
80
+ }
81
+ return (found ?? 'schema') === kind;
68
82
  }
69
- function search(cwd, ceiling) {
83
+ function search(cwd, ceiling, spec) {
70
84
  const budget = { left: MAX_VISITS, seen: new Set() };
71
85
  let dir = cwd;
72
86
  for (let level = 0; level < MAX_LEVELS; level++) {
73
- const found = nearest(dir, budget);
87
+ const found = nearest(dir, budget, spec.kind);
74
88
  if (found.length === 1) {
75
89
  return found[0];
76
90
  }
77
91
  if (found.length > 1) {
78
- throw new CliError(`more than one index is equally close to here: ${found.join(', ')}`, EXIT.usage, `say which with --dir, or set ${DIR_ENV}`);
92
+ throw new CliError(`more than one index is equally close to here: ${found.join(', ')}`, EXIT.usage, `say which with --dir, or set ${spec.envName}`);
79
93
  }
80
94
  budget.seen.add(dir);
81
95
  const up = dirname(dir);
@@ -87,10 +101,10 @@ function search(cwd, ceiling) {
87
101
  return undefined;
88
102
  }
89
103
  /** Every index at the shallowest depth that has any, so a tie can be reported as one. */
90
- function nearest(root, budget) {
104
+ function nearest(root, budget, kind) {
91
105
  let frontier = [root];
92
106
  for (let depth = 0; depth <= MAX_DEPTH && frontier.length > 0; depth++) {
93
- const found = frontier.filter(isIndex);
107
+ const found = frontier.filter((dir) => isIndex(dir, kind));
94
108
  if (found.length > 0) {
95
109
  return found;
96
110
  }
@@ -0,0 +1,50 @@
1
+ export declare const MANIFEST_FILE = "manifest.json";
2
+ export type IndexKind = 'schema' | 'docs';
3
+ /** How to name a kind when something has to be said about it. */
4
+ export declare const SUBJECT: Record<IndexKind, {
5
+ label: string;
6
+ command: string;
7
+ }>;
8
+ /** The part of a manifest that does not depend on what was indexed. */
9
+ export interface IndexHead {
10
+ /** the format version of this kind of index */
11
+ version: number;
12
+ kind: IndexKind;
13
+ createdAt: string;
14
+ indexer: string;
15
+ /** `ref` as it was typed, `id` as the embedder answers to it */
16
+ embedding: {
17
+ ref: string;
18
+ id: string;
19
+ dimensions: number;
20
+ };
21
+ /** whether the table carries an fts index, and whether it carries a vector one */
22
+ indexes: {
23
+ fts: boolean;
24
+ vector: boolean;
25
+ };
26
+ }
27
+ /** One subject's identity: enough to find an index, read one, and refuse one. */
28
+ export interface IndexSpec {
29
+ kind: IndexKind;
30
+ version: number;
31
+ /** the name a NEW index is given; nothing ever searches for it */
32
+ defaultDir: string;
33
+ envName: string;
34
+ }
35
+ /**
36
+ * The manifest, or the reason there is not one. Three refusals, and each names
37
+ * a different thing to do next: build one, use the other subject, rebuild.
38
+ */
39
+ export declare function readHead<T extends IndexHead>(dir: string, spec: IndexSpec): Promise<T>;
40
+ /**
41
+ * A store answers with the neighbours of a vector, and a vector means nothing
42
+ * without the model that produced it. Asking one model's index a question
43
+ * embedded by another returns rows, in an order that is noise.
44
+ *
45
+ * Either spelling is accepted, because `openai:text-embedding-3-small` and
46
+ * `text-embedding-3-small` are one model and which of them was typed is not
47
+ * something anyone should have to remember.
48
+ */
49
+ export declare function assertSameEmbedding(head: IndexHead, ref: string): void;
50
+ //# sourceMappingURL=manifest.d.ts.map
@@ -0,0 +1,62 @@
1
+ import { CliError, EXIT } from '@zenera/cli/lib';
2
+ import { readFile } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
+ // ---------------------------------------------------------------------------
5
+ // What every index says about itself
6
+ //
7
+ // An index is a directory, and `manifest.json` is the first thing anything
8
+ // reads out of one. It is written LAST by a build, so its presence is the
9
+ // commit marker: a half-built directory has no manifest and reads as "not
10
+ // indexed" rather than as a store that quietly lost half of what it holds.
11
+ //
12
+ // `kind` is what keeps two subjects apart in one tree. Without it, pointing a
13
+ // document search at an API index is not an error — it is rows, in an order
14
+ // that means nothing, about the wrong corpus entirely. That is the failure
15
+ // worth spending a field on.
16
+ // ---------------------------------------------------------------------------
17
+ export const MANIFEST_FILE = 'manifest.json';
18
+ /** How to name a kind when something has to be said about it. */
19
+ export const SUBJECT = {
20
+ schema: { label: 'a schema index', command: 'zen rag schema' },
21
+ docs: { label: 'a document index', command: 'zen rag docs' },
22
+ };
23
+ /**
24
+ * The manifest, or the reason there is not one. Three refusals, and each names
25
+ * a different thing to do next: build one, use the other subject, rebuild.
26
+ */
27
+ export async function readHead(dir, spec) {
28
+ const { label, command } = SUBJECT[spec.kind];
29
+ let text;
30
+ try {
31
+ text = await readFile(join(dir, MANIFEST_FILE), 'utf8');
32
+ }
33
+ catch {
34
+ throw new CliError(`${dir} does not hold an index`, EXIT.invalid, `build one with \`${command} index\`, or name an existing one with --dir or $${spec.envName}`);
35
+ }
36
+ const head = JSON.parse(text);
37
+ // Written since there was more than one kind; before that there was only
38
+ // the one, so a manifest that does not say is a schema index.
39
+ const kind = head.kind ?? 'schema';
40
+ if (kind !== spec.kind) {
41
+ throw new CliError(`${dir} holds ${SUBJECT[kind].label}, not ${label}`, EXIT.invalid, `read it with \`${SUBJECT[kind].command} search\``);
42
+ }
43
+ if (head.version !== spec.version) {
44
+ throw new CliError(`${dir} is a version ${head.version} index, and this indexer reads version ${spec.version}`, EXIT.invalid, `rebuild it with \`${command} index\``);
45
+ }
46
+ return head;
47
+ }
48
+ /**
49
+ * A store answers with the neighbours of a vector, and a vector means nothing
50
+ * without the model that produced it. Asking one model's index a question
51
+ * embedded by another returns rows, in an order that is noise.
52
+ *
53
+ * Either spelling is accepted, because `openai:text-embedding-3-small` and
54
+ * `text-embedding-3-small` are one model and which of them was typed is not
55
+ * something anyone should have to remember.
56
+ */
57
+ export function assertSameEmbedding(head, ref) {
58
+ if (ref !== head.embedding.ref && ref !== head.embedding.id) {
59
+ throw new CliError(`this index was built with ${head.embedding.ref}, not ${ref}`, EXIT.invalid, `search it with --embedding ${head.embedding.ref}, or rebuild it`);
60
+ }
61
+ }
62
+ //# sourceMappingURL=manifest.js.map
@@ -0,0 +1,57 @@
1
+ export declare const LOCK_FILE = ".lock";
2
+ export declare const README_FILE = "README.md";
3
+ /** The floor on how often README.md is rewritten. */
4
+ export declare const INTERVAL_MS = 5000;
5
+ export interface Building<S> {
6
+ documents: readonly string[];
7
+ embedding: string;
8
+ started: number;
9
+ now: number;
10
+ /** the current phase, as it should be said out loud */
11
+ step: string;
12
+ done: number;
13
+ total: number;
14
+ /** what the documents turned out to hold, once they have been read */
15
+ summary: S | undefined;
16
+ }
17
+ export interface Completed<M> {
18
+ dir: string;
19
+ manifest: M;
20
+ ms: number;
21
+ }
22
+ export interface Failed {
23
+ documents: readonly string[];
24
+ step: string;
25
+ reason: unknown;
26
+ started: number;
27
+ }
28
+ /** The three states a README can be in, written by whoever knows the subject. */
29
+ export interface Report<S, M> {
30
+ building(state: Building<S>): string;
31
+ complete(state: Completed<M>): string;
32
+ failed(state: Failed): string;
33
+ }
34
+ export interface BuildPlan<S, M, P extends string> {
35
+ /** where the index goes */
36
+ dir: string;
37
+ files: readonly string[];
38
+ /** the embedding reference as it was typed */
39
+ embedding: string;
40
+ indexer: string;
41
+ /** every phase, in order; the first is where a build starts */
42
+ phases: Readonly<Record<P, string>>;
43
+ report: Report<S, M>;
44
+ }
45
+ export interface Journal<S, M, P extends string> {
46
+ phase(name: P): void;
47
+ read(summary: S, total: number): void;
48
+ progress(done: number, total: number): void;
49
+ finish(manifest: M): void;
50
+ fail(reason: unknown): void;
51
+ }
52
+ /**
53
+ * Takes the directory, or refuses it. Two builds writing one index would
54
+ * interleave their store writes and leave one that neither of them describes.
55
+ */
56
+ export declare function beginBuild<S, M, P extends string>(plan: BuildPlan<S, M, P>): Journal<S, M, P>;
57
+ //# sourceMappingURL=progress.d.ts.map
@@ -0,0 +1,155 @@
1
+ import { CliError, EXIT } from '@zenera/cli/lib';
2
+ import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
3
+ import { hostname } from 'node:os';
4
+ import { basename, join } from 'node:path';
5
+ // ---------------------------------------------------------------------------
6
+ // A build that says what it is doing
7
+ //
8
+ // Indexing a large corpus is minutes of silence with a directory slowly
9
+ // filling up, and the directory is the only thing a second person — or a second
10
+ // process, or an agent reading the tree — ever sees. So the build writes two
11
+ // files into it and keeps them true:
12
+ //
13
+ // .lock who is building this, right now. Gone when nothing is.
14
+ // README.md a progress report while it runs, and a description of what the
15
+ // index holds once it does not.
16
+ //
17
+ // Nothing in either file is an absolute path. A project's `assets/` directory
18
+ // is mounted at `/assets` inside an agent's sandbox, so an index built there is
19
+ // read under a name this process never sees; a host path would be a lie there.
20
+ //
21
+ // What is here is the mechanics — the lock, the throttle, the atomic rewrite.
22
+ // The words belong to whatever is being indexed, so they arrive as a `Report`.
23
+ // ---------------------------------------------------------------------------
24
+ export const LOCK_FILE = '.lock';
25
+ export const README_FILE = 'README.md';
26
+ /** The floor on how often README.md is rewritten. */
27
+ export const INTERVAL_MS = 5000;
28
+ /**
29
+ * Takes the directory, or refuses it. Two builds writing one index would
30
+ * interleave their store writes and leave one that neither of them describes.
31
+ */
32
+ export function beginBuild(plan) {
33
+ const documents = plan.files.map((file) => basename(file));
34
+ const started = Date.now();
35
+ const lock = {
36
+ pid: process.pid,
37
+ host: hostname(),
38
+ startedAt: new Date(started).toISOString(),
39
+ indexer: plan.indexer,
40
+ embedding: plan.embedding,
41
+ documents,
42
+ };
43
+ mkdirSync(plan.dir, { recursive: true });
44
+ claim(join(plan.dir, LOCK_FILE), lock);
45
+ let phase = Object.keys(plan.phases)[0];
46
+ let summary;
47
+ let done = 0;
48
+ let total = 0;
49
+ let wroteAt = 0;
50
+ let closed = false;
51
+ const write = (body) => {
52
+ // Through a temp name, so a reader never catches half a file.
53
+ const target = join(plan.dir, README_FILE);
54
+ const temp = `${target}.tmp`;
55
+ writeFileSync(temp, body);
56
+ renameSync(temp, target);
57
+ wroteAt = Date.now();
58
+ };
59
+ const report = () => write(plan.report.building({
60
+ documents,
61
+ embedding: plan.embedding,
62
+ started,
63
+ now: Date.now(),
64
+ step: plan.phases[phase],
65
+ done,
66
+ total,
67
+ summary,
68
+ }));
69
+ const maybe = () => {
70
+ if (!closed && Date.now() - wroteAt >= INTERVAL_MS) {
71
+ report();
72
+ }
73
+ };
74
+ // The first embedding request can block for ten seconds or more, so a purely
75
+ // event-driven throttle would leave the elapsed line frozen through it.
76
+ const ticker = setInterval(maybe, INTERVAL_MS);
77
+ ticker.unref();
78
+ const close = (body) => {
79
+ if (closed) {
80
+ return;
81
+ }
82
+ closed = true;
83
+ clearInterval(ticker);
84
+ write(body);
85
+ rmSync(join(plan.dir, LOCK_FILE), { force: true });
86
+ };
87
+ report();
88
+ return {
89
+ phase(name) {
90
+ phase = name;
91
+ maybe();
92
+ },
93
+ read(seen, count) {
94
+ summary = seen;
95
+ total = count;
96
+ maybe();
97
+ },
98
+ progress(at, of) {
99
+ done = at;
100
+ total = of;
101
+ maybe();
102
+ },
103
+ finish(manifest) {
104
+ close(plan.report.complete({ dir: plan.dir, manifest, ms: Date.now() - started }));
105
+ },
106
+ fail(reason) {
107
+ close(plan.report.failed({ documents, step: plan.phases[phase], reason, started }));
108
+ },
109
+ };
110
+ }
111
+ /**
112
+ * `wx` makes the create and the check one operation, so two builds racing for
113
+ * the same directory cannot both win. A lock whose process is gone is stale by
114
+ * definition and is taken rather than respected — a crashed build must not make
115
+ * a directory permanently unbuildable.
116
+ */
117
+ function claim(path, lock) {
118
+ const body = `${JSON.stringify(lock, null, 4)}\n`;
119
+ try {
120
+ writeFileSync(path, body, { flag: 'wx' });
121
+ return;
122
+ }
123
+ catch (err) {
124
+ if (err.code !== 'EEXIST') {
125
+ throw err;
126
+ }
127
+ }
128
+ const held = readLock(path);
129
+ if (held && held.host === hostname() && alive(held.pid)) {
130
+ throw new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`);
131
+ }
132
+ writeFileSync(path, body);
133
+ }
134
+ function readLock(path) {
135
+ try {
136
+ return JSON.parse(readFileSync(path, 'utf8'));
137
+ }
138
+ catch {
139
+ return undefined;
140
+ }
141
+ }
142
+ /**
143
+ * `kill(pid, 0)` sends no signal and only asks whether the process exists.
144
+ * EPERM means it exists and belongs to someone else, which still counts.
145
+ */
146
+ function alive(pid) {
147
+ try {
148
+ process.kill(pid, 0);
149
+ return true;
150
+ }
151
+ catch (err) {
152
+ return err.code === 'EPERM';
153
+ }
154
+ }
155
+ //# sourceMappingURL=progress.js.map
@@ -0,0 +1,13 @@
1
+ /** An indented block, which markdown renders verbatim and an agent reads as a table. */
2
+ export declare function fields(rows: readonly string[][]): string[];
3
+ /** A markdown table, padded so the source is readable unrendered. */
4
+ export declare function grid(headers: readonly string[], rows: readonly (readonly string[])[]): string[];
5
+ export declare function plural(n: number, one: string, many?: string): string;
6
+ export declare function duration(ms: number): string;
7
+ /** The first line only: a stack trace in a README helps nobody. */
8
+ export declare function message(reason: unknown): string;
9
+ export declare function searched(indexes: {
10
+ fts: boolean;
11
+ vector: boolean;
12
+ }): string;
13
+ //# sourceMappingURL=prose.d.ts.map
@@ -0,0 +1,56 @@
1
+ // ---------------------------------------------------------------------------
2
+ // Small renderings, shared by every README a build writes
3
+ //
4
+ // These are markdown that an agent reads as often as a person does, so the
5
+ // phrasing is plain and the shapes are stable. Nothing here knows what was
6
+ // indexed.
7
+ // ---------------------------------------------------------------------------
8
+ /** An indented block, which markdown renders verbatim and an agent reads as a table. */
9
+ export function fields(rows) {
10
+ const width = Math.max(...rows.map((r) => (r[0] ?? '').length));
11
+ return rows.map(([name, value]) => ` ${(name ?? '').padEnd(width)} ${value ?? ''}`);
12
+ }
13
+ /** A markdown table, padded so the source is readable unrendered. */
14
+ export function grid(headers, rows) {
15
+ const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? '').length)));
16
+ const line = (cells) => `| ${cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(' | ')} |`;
17
+ return [
18
+ line(headers),
19
+ `| ${widths.map((w) => '-'.repeat(w)).join(' | ')} |`,
20
+ ...rows.map(line),
21
+ ];
22
+ }
23
+ export function plural(n, one, many = `${one}s`) {
24
+ return `${n} ${n === 1 ? one : many}`;
25
+ }
26
+ export function duration(ms) {
27
+ const seconds = Math.round(ms / 1000);
28
+ if (seconds < 1) {
29
+ return 'under a second';
30
+ }
31
+ if (seconds < 60) {
32
+ return `${seconds}s`;
33
+ }
34
+ const minutes = Math.floor(seconds / 60);
35
+ return minutes < 60
36
+ ? `${minutes}m${seconds % 60}s`
37
+ : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
38
+ }
39
+ /** The first line only: a stack trace in a README helps nobody. */
40
+ export function message(reason) {
41
+ const text = reason instanceof Error ? reason.message : String(reason);
42
+ return text.split('\n')[0]?.trim() || 'no reason given';
43
+ }
44
+ export function searched(indexes) {
45
+ if (!indexes.fts && !indexes.vector) {
46
+ return 'scanned flat: neither index was built';
47
+ }
48
+ if (!indexes.vector) {
49
+ // Below a couple of thousand rows an IVF index has nothing to train on.
50
+ return 'searched by full text and by vector, the latter as a flat scan';
51
+ }
52
+ return indexes.fts
53
+ ? 'searched by full text and by vector'
54
+ : 'searched by vector, with no full-text index';
55
+ }
56
+ //# sourceMappingURL=prose.js.map
package/dist/index.d.ts CHANGED
@@ -1,13 +1,14 @@
1
- export * from './present.ts';
2
- export * from './query.ts';
1
+ export * from './common/locate.ts';
2
+ export * from './common/manifest.ts';
3
+ export * from './common/match.ts';
3
4
  export * from './schema/build.ts';
4
5
  export * from './schema/entities.ts';
5
6
  export * from './schema/files.ts';
6
7
  export * from './schema/graph.ts';
7
8
  export * from './schema/hydrate.ts';
8
- export * from './schema/locate.ts';
9
9
  export * from './schema/lookup.ts';
10
- export * from './schema/match.ts';
10
+ export * from './schema/present.ts';
11
+ export * from './schema/query.ts';
11
12
  export * from './schema/render.ts';
12
13
  export * from './schema/schema.ts';
13
14
  export * from './schema/search.ts';
@@ -15,5 +16,6 @@ export * from './schema/spec.ts';
15
16
  export * from './schema/store.ts';
16
17
  export * from './schema/subgraph.ts';
17
18
  export * from './schema/tools.ts';
19
+ export * from './schema/trace.ts';
18
20
  export * from './schema/typescript.ts';
19
21
  //# sourceMappingURL=index.d.ts.map