@fro.bot/systematic 3.29.1 → 3.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -20,12 +20,13 @@ import {
20
20
  discoverSkills,
21
21
  SEARCH_TOOL_NAME,
22
22
  SEARCH_ARGS,
23
+ KIND_SELECTOR_EXTERNAL,
23
24
  SEARCH_TOOL_DESCRIPTION,
24
25
  SEARCH_ARG_DESCRIPTIONS,
25
26
  SEARCH_NOTICES,
26
27
  SEARCH_CONFIG_UNREADABLE_NOTE,
27
28
  runSearch
28
- } from "./index-e467g4b6.js";
29
+ } from "./index-jvpbpj9z.js";
29
30
 
30
31
  // src/index.ts
31
32
  import path8 from "path";
@@ -982,11 +983,12 @@ function normalizePath(path) {
982
983
  function isSearchEnabled(settings) {
983
984
  return settings?.enabled !== false;
984
985
  }
985
- async function executeSearch(args, startDir, settings, unreadable) {
986
+ async function executeSearch(args, startDir, settings, unreadable, configOptions) {
986
987
  try {
987
988
  if (startDir === undefined)
988
989
  return SEARCH_NOTICES.noProject;
989
990
  const record = isRecord(args) ? args : {};
991
+ const external = record[SEARCH_ARGS.kind] === KIND_SELECTOR_EXTERNAL;
990
992
  const response = await runSearch({
991
993
  query: record[SEARCH_ARGS.query],
992
994
  kind: record[SEARCH_ARGS.kind],
@@ -994,7 +996,8 @@ async function executeSearch(args, startDir, settings, unreadable) {
994
996
  }, {
995
997
  startDir,
996
998
  settings,
997
- ...unreadable === true && { notes: [SEARCH_CONFIG_UNREADABLE_NOTE] }
999
+ configOptions,
1000
+ ...unreadable === true && !external && { notes: [SEARCH_CONFIG_UNREADABLE_NOTE] }
998
1001
  });
999
1002
  return response.text;
1000
1003
  } catch {
@@ -1005,7 +1008,7 @@ function usableDirectory(value) {
1005
1008
  return typeof value === "string" && value.length > 0 ? value : undefined;
1006
1009
  }
1007
1010
  function createSearchTool(options) {
1008
- const { settings, unreadable } = options;
1011
+ const { settings, unreadable, configOptions } = options;
1009
1012
  return {
1010
1013
  description: SEARCH_TOOL_DESCRIPTION,
1011
1014
  args: {
@@ -1014,7 +1017,7 @@ function createSearchTool(options) {
1014
1017
  [SEARCH_ARGS.limit]: number().int().optional().describe(SEARCH_ARG_DESCRIPTIONS.limit)
1015
1018
  },
1016
1019
  async execute(args, context) {
1017
- return executeSearch(args, usableDirectory(context.directory), settings, unreadable);
1020
+ return executeSearch(args, usableDirectory(context.directory), settings, unreadable, configOptions);
1018
1021
  }
1019
1022
  };
1020
1023
  }
@@ -18,5 +18,5 @@
18
18
  * reason it exists as a separate file rather than living directly in
19
19
  * `config.ts`.
20
20
  */
21
- export declare const CONFIG_PROTECTED_FIELD_PATHS: readonly ['profiles', 'allow_project_profiles', 'search', 'agents.*.model', 'agents.*.permission', 'agents.*.skills', 'agents.*.variant', 'agents.*.opencode', 'agents.*.pi', 'categories.*.model', 'categories.*.permission', 'categories.*.skills', 'categories.*.variant', 'categories.*.opencode', 'categories.*.pi'];
21
+ export declare const CONFIG_PROTECTED_FIELD_PATHS: readonly ['profiles', 'allow_project_profiles', 'search.enabled', 'search.allow_project_sources', 'agents.*.model', 'agents.*.permission', 'agents.*.skills', 'agents.*.variant', 'agents.*.opencode', 'agents.*.pi', 'categories.*.model', 'categories.*.permission', 'categories.*.skills', 'categories.*.variant', 'categories.*.opencode', 'categories.*.pi'];
22
22
  export type ConfigProtectedFieldPath = (typeof CONFIG_PROTECTED_FIELD_PATHS)[number];
@@ -387,6 +387,16 @@ export declare function createSystematicConfigSchema(opts: SystematicConfigSchem
387
387
  allow_project_profiles: z.ZodDefault<z.ZodBoolean>;
388
388
  search: z.ZodOptional<z.ZodObject<{
389
389
  enabled: z.ZodOptional<z.ZodBoolean>;
390
+ allow_project_sources: z.ZodOptional<z.ZodArray<z.ZodString>>;
391
+ sources: z.ZodOptional<z.ZodArray<z.ZodObject<{
392
+ name: z.ZodString;
393
+ format: z.ZodEnum<{
394
+ deciduous: "deciduous";
395
+ markdown: "markdown";
396
+ }>;
397
+ roots: z.ZodArray<z.ZodString>;
398
+ projects: z.ZodOptional<z.ZodArray<z.ZodString>>;
399
+ }, z.core.$strict>>>;
390
400
  }, z.core.$strict>>;
391
401
  disabled_skills: z.ZodDefault<z.ZodArray<z.ZodEnum<{
392
402
  [x: string]: string;
@@ -585,6 +595,16 @@ export declare const SystematicConfigSchema: z.ZodObject<{
585
595
  allow_project_profiles: z.ZodDefault<z.ZodBoolean>;
586
596
  search: z.ZodOptional<z.ZodObject<{
587
597
  enabled: z.ZodOptional<z.ZodBoolean>;
598
+ allow_project_sources: z.ZodOptional<z.ZodArray<z.ZodString>>;
599
+ sources: z.ZodOptional<z.ZodArray<z.ZodObject<{
600
+ name: z.ZodString;
601
+ format: z.ZodEnum<{
602
+ deciduous: "deciduous";
603
+ markdown: "markdown";
604
+ }>;
605
+ roots: z.ZodArray<z.ZodString>;
606
+ projects: z.ZodOptional<z.ZodArray<z.ZodString>>;
607
+ }, z.core.$strict>>>;
588
608
  }, z.core.$strict>>;
589
609
  disabled_skills: z.ZodDefault<z.ZodArray<z.ZodEnum<{
590
610
  [x: string]: string;
@@ -1,6 +1,6 @@
1
1
  import { CONFIG_PROTECTED_FIELD_PATHS, type ConfigProtectedFieldPath } from './config-protected-fields.js';
2
2
  import { type RoutingTarget } from './routing-resolver.js';
3
- import type { SearchSettings } from './search-core.js';
3
+ import { type SearchSchemaOutput } from './search-user-config.js';
4
4
  export interface BootstrapConfig {
5
5
  enabled: boolean;
6
6
  file?: string;
@@ -174,11 +174,14 @@ export interface SystematicConfig {
174
174
  skills_as_commands: boolean;
175
175
  allow_project_profiles: boolean;
176
176
  /**
177
- * Merged `search` settings from user and custom config only. Absent when
178
- * neither defines one: there is no default, and the core applies its own at
179
- * use time.
177
+ * Merged `search` settings from user and custom config only: `enabled`
178
+ * (last-defining), `allow_project_sources` (union) and the user `sources`
179
+ * (by name, custom winning). Absent when neither defines one: there is no
180
+ * default, and the core applies its own at use time. A project config's
181
+ * `search.sources` are validated at load and then discarded; they never
182
+ * appear here. The per-call external reader is their only consumer.
180
183
  */
181
- search?: SearchSettings;
184
+ search?: SearchSchemaOutput;
182
185
  /**
183
186
  * Set (to `true`) when a user or custom `search` block, or its file, could
184
187
  * not be read. That block is ignored and search results carry a
@@ -0,0 +1,38 @@
1
+ /**
2
+ * The one bounded file read behind every search reader that opens a file whose
3
+ * content an untrusted checkout or folder controls: the project config, the
4
+ * project identity files and the corpus and external source files.
5
+ *
6
+ * Import rule: node builtins only. The `systematic-search` bin bundles it, so
7
+ * it must never import `config.ts`, `config-schema.ts` or any module that does.
8
+ */
9
+ export type BoundedRead = {
10
+ readonly kind: 'bytes';
11
+ readonly bytes: Buffer;
12
+ }
13
+ /** The path does not exist (`ENOENT`). */
14
+ | {
15
+ readonly kind: 'absent';
16
+ }
17
+ /** The opened descriptor is not a regular file, such as a directory or a FIFO. */
18
+ | {
19
+ readonly kind: 'not-regular';
20
+ }
21
+ /** The content reached `maxBytes + 1`; `bytesRead` is what was actually read. */
22
+ | {
23
+ readonly kind: 'too-large';
24
+ readonly bytesRead: number;
25
+ }
26
+ /** Any other open or read failure. */
27
+ | {
28
+ readonly kind: 'failed';
29
+ };
30
+ /**
31
+ * Read a file through an open fd, never more than `maxBytes + 1` bytes.
32
+ * Reaching `maxBytes + 1` rejects the file, so a file that is already too
33
+ * large and one that grows past the cap while it is being read fail the same
34
+ * way, and the size is never trusted from a `stat`. The open is non-blocking
35
+ * and the fd must be a regular file, so a FIFO or device planted at the path
36
+ * cannot hang the reader. Never throws.
37
+ */
38
+ export declare function readBytesBounded(filePath: string, maxBytes: number): BoundedRead;
@@ -13,9 +13,10 @@
13
13
  * failure; 2 for usage errors. There is no `--json`.
14
14
  *
15
15
  * Import rule: this module is bundled into the `systematic-search` bin, so it
16
- * may import only `search-core.ts` and node builtins.
16
+ * may import only `search-core.ts`, node builtins and types.
17
17
  */
18
18
  import { type SearchSettings } from './search-core.js';
19
+ import type { UserLevelConfigDirOptions } from './search-user-config.js';
19
20
  /**
20
21
  * The usage text for one surface. `commandName` is what that surface's users
21
22
  * type (`systematic search` or `systematic-search`), so the text never tells a
@@ -24,6 +25,12 @@ import { type SearchSettings } from './search-core.js';
24
25
  export declare function searchCliUsage(commandName: string): string;
25
26
  /** What `search.enabled: false` prints. The researcher falls back to grep on it. */
26
27
  export declare const SEARCH_CLI_DISABLED_NOTICE = "Search is disabled (search.enabled is false in your Systematic config), so there is nothing to search.";
28
+ /**
29
+ * Printed once to stderr when `--kind external` finds the project's config
30
+ * file structurally broken. Fixed text: no path, no value. The reply
31
+ * carries the aggregate line and the bounded warnings.
32
+ */
33
+ export declare const SEARCH_CLI_PROJECT_CONFIG_WARNING = "[systematic] The project Systematic config could not be read, so it proposes no external sources.";
27
34
  /** What a settings loader returns. The bin's user-level reader returns the same shape. */
28
35
  export interface SearchCliSettingsLoad {
29
36
  readonly settings?: SearchSettings | undefined;
@@ -41,6 +48,12 @@ export interface SearchCliOptions {
41
48
  /** The command the usage text names: `systematic search` or `systematic-search`. */
42
49
  readonly usageName: string;
43
50
  readonly loadSettings: SearchCliSettingsLoader;
51
+ /**
52
+ * The user-level config directories `--kind external` reads its allow list
53
+ * and user sources from. The caller passes the same options it gives its
54
+ * settings loader, so one override reaches both.
55
+ */
56
+ readonly configOptions?: UserLevelConfigDirOptions | undefined;
44
57
  readonly outputSink?: (message: string) => void;
45
58
  readonly errorSink?: (message: string) => void;
46
59
  }
@@ -8,6 +8,12 @@
8
8
  * I/O: the start directory, the settings, and mapping the response to its
9
9
  * host's return shape. Adapters never format text themselves.
10
10
  *
11
+ * `kind: "external"` is a selector beside the four kinds, not a fifth kind. It
12
+ * takes its own branch: the core resolves the active sources for the call
13
+ * through an injectable resolver, loads each one, ranks each source on its own
14
+ * index, and hands the sections to `renderExternalResult`. Built-in kinds never
15
+ * reach that branch, so their output is unchanged.
16
+ *
11
17
  * This module also owns the `SearchSettings` type, the tool name and argument
12
18
  * constants, and the lifecycle constants and resolver. The resolver follows
13
19
  * the learnings-researcher's rules: at most 5 hops, a visited set, and a form
@@ -18,8 +24,11 @@
18
24
  * `Caution: successor unresolved`. Nothing is dropped.
19
25
  */
20
26
  import { type CorpusDocument, type CorpusLoad, type DocumentKind } from './search-corpus.js';
27
+ import { type SourceLoad } from './search-external.js';
28
+ import { type ActiveSource, type ExternalSourcesInput, type ExternalSourcesResolution } from './search-external-sources.js';
21
29
  import { type Bm25fParams } from './search-lexical.js';
22
- import { type SearchHitStatus, type SearchResult, SUCCESSOR_LABEL, SUPERSEDES_LABEL } from './search-render.js';
30
+ import { type ExternalFixedReply, type ExternalSearchResult, type ExternalSection, type SearchHitStatus, type SearchResult, SUCCESSOR_LABEL, SUPERSEDES_LABEL } from './search-render.js';
31
+ import { type UserLevelConfigDirOptions } from './search-user-config.js';
23
32
  export type { SearchHit, SearchResult } from './search-render.js';
24
33
  export { SUCCESSOR_LABEL, SUPERSEDES_LABEL };
25
34
  /** Tool name shared by every host surface. */
@@ -36,16 +45,24 @@ export declare const MAX_LIMIT = 20;
36
45
  export declare const MAX_QUERY_CHARS = 2000;
37
46
  /** The `kind` value that searches every kind in one ranking. It is a selector, not a table key. */
38
47
  export declare const KIND_SELECTOR_ALL = "all";
48
+ /**
49
+ * The `kind` value that searches the user-allowed external sources. Like `all`
50
+ * it is a selector, not a table key, and it is exclusive: it does not combine
51
+ * with a kind or with `all`.
52
+ */
53
+ export declare const KIND_SELECTOR_EXTERNAL = "external";
54
+ /** Most hits one external source returns, whatever `limit` asks for. */
55
+ export declare const MAX_EXTERNAL_HITS_PER_SOURCE = 5;
39
56
  /**
40
57
  * The tool text every host surface shows the model. `limit` is a plain integer
41
58
  * in every host schema, so its range is stated here and nowhere else: the core
42
59
  * owns the out-of-range error and a host never pre-empts it.
43
60
  */
44
- export declare const SEARCH_TOOL_DESCRIPTION = "Search this project's durable outputs by symptom, error text, module or topic: the docs/solutions learnings, and the docs/plans, docs/brainstorms and docs/ideation documents that earlier work wrote. Returns ranked hits with path, title, kind, snippet, tags and status. A superseded document is replaced by its successor in the same kind, a successor in another kind is named on a Successor line, and a stale one carries a caution. Read a hit's path for the full text. Use it before debugging a failure or planning work that a past learning, plan, brainstorm or ideation doc may already cover.";
61
+ export declare const SEARCH_TOOL_DESCRIPTION = "Search this project's durable outputs by symptom, error text, module or topic: the docs/solutions learnings, and the docs/plans, docs/brainstorms and docs/ideation documents that earlier work wrote. Returns ranked hits with path, title, kind, snippet, tags and status. A superseded document is replaced by its successor in the same kind, a successor in another kind is named on a Successor line, and a stale one carries a caution. Read a hit's path for the full text. Use it before debugging a failure or planning work that a past learning, plan, brainstorm or ideation doc may already cover. With kind external, a hit's path is `<source>/<path>`: the source name, then a path inside one of that source's roots, which the source's section header lists. Strip the source name to resolve it. The reply does not say which root a hit came from; with several roots, the file is under the first listed root that has it.";
45
62
  export declare const SEARCH_ARG_DESCRIPTIONS: {
46
63
  readonly query: "What to look for: a symptom, error message, module or topic, in plain words. At most 2000 characters.";
47
- readonly kind: `Optional. The kind of durable output to search: ${string}, or "all" for every kind in one ranking. Defaults to "solution".`;
48
- readonly limit: "Optional. How many hits to return, an integer from 1 to 20. Defaults to 8.";
64
+ readonly kind: `Optional. The kind of durable output to search: ${string}, or "all" for every kind in one ranking, or "external" for the external decision sources the user allowed, each ranked on its own. Defaults to "solution".`;
65
+ readonly limit: "Optional. How many hits to return, an integer from 1 to 20. Defaults to 8. With kind \"external\" it applies to each source instead, and at most 5 hits per source are returned.";
49
66
  };
50
67
  /** One-line texts an adapter returns when it cannot reach the core. They carry no detail from the failure. */
51
68
  export declare const SEARCH_NOTICES: {
@@ -86,11 +103,56 @@ export interface SearchOptions {
86
103
  readonly lexicalParams?: Bm25fParams | undefined;
87
104
  /** Replace the disk loader. Tests use this to control load order. */
88
105
  readonly corpusLoader?: ((root: string, kind: DocumentKind) => CorpusLoad) | undefined;
106
+ /**
107
+ * The user-level config directories `kind: "external"` reads its allow list
108
+ * and user sources from. A host or CLI config-dir override goes here; left
109
+ * out, the reader uses the home directory and `OPENCODE_CONFIG_DIR`.
110
+ */
111
+ readonly configOptions?: UserLevelConfigDirOptions | undefined;
112
+ /**
113
+ * Replace the per-call external reader. The default resolves the active
114
+ * sources from the user files, the project file and the project identity.
115
+ * Tests use this to control what a call sees.
116
+ */
117
+ readonly externalResolver?: ExternalResolver | undefined;
89
118
  }
119
+ /** Reads everything for one `external` call and returns the active sources. The default never throws. */
120
+ export type ExternalResolver = (input: ExternalSourcesInput) => ExternalSourcesResolution;
121
+ /** The default resolver: the per-call reader in `search-external-sources.ts`. */
122
+ export declare const defaultExternalResolver: ExternalResolver;
123
+ /**
124
+ * What an `external` call produced: one of the core's fixed replies, or the
125
+ * ranked sections of the active sources.
126
+ */
127
+ export type ExternalOutcome = {
128
+ readonly kind: 'fixed';
129
+ readonly reply: ExternalFixedReply;
130
+ } | {
131
+ readonly kind: 'sections';
132
+ readonly result: ExternalSearchResult;
133
+ };
134
+ /**
135
+ * A response is tagged by `ok`, then by `selector`: a built-in response carries
136
+ * `result` and an external one carries `external`, and neither carries the
137
+ * other, so a consumer must narrow on `selector` before reading either. Every
138
+ * response carries `text`, which is all the adapters read.
139
+ */
90
140
  export type SearchResponse = {
91
141
  readonly ok: true;
142
+ readonly selector: 'builtin';
92
143
  readonly text: string;
93
144
  readonly result: SearchResult;
145
+ } | {
146
+ readonly ok: true;
147
+ readonly selector: 'external';
148
+ readonly text: string;
149
+ readonly external: ExternalOutcome;
150
+ /**
151
+ * Present (and `true`) when the project's config file had a structural
152
+ * error, so it proposed nothing. The reply text already says so; the CLI
153
+ * and the bin also print one stderr warning for it.
154
+ */
155
+ readonly projectConfigUnreadable?: true;
94
156
  } | {
95
157
  readonly ok: false;
96
158
  readonly text: string;
@@ -144,3 +206,16 @@ export declare function lifecycleLines(entry: ResolvedEntry): string[];
144
206
  * request ranks exactly as it does alone.
145
207
  */
146
208
  export declare function runSearch(request: SearchRequest, options: SearchOptions): Promise<SearchResponse>;
209
+ interface ActiveLoad {
210
+ readonly source: ActiveSource;
211
+ readonly load: SourceLoad;
212
+ }
213
+ /**
214
+ * Rank every source on its own and build the sections, ordered by name in
215
+ * code-unit order (then origin). Scores are not comparable across sources, so
216
+ * each source gets its own lexical index: `title` to title, `node_type` to
217
+ * tags and `body` to body. Hits within a section come best first with ties by
218
+ * path, capped at `limit` and at `MAX_EXTERNAL_HITS_PER_SOURCE`. A source that
219
+ * loaded no record is a state-only section.
220
+ */
221
+ export declare function rankExternalSources(entries: readonly ActiveLoad[], queryTerms: readonly string[], limit: number, params?: Bm25fParams): ExternalSection[];
@@ -9,6 +9,7 @@
9
9
  * carry the narrowed `status`, `superseded_by` and `stale_reason` that the
10
10
  * search core's resolver reads.
11
11
  */
12
+ import fs from 'node:fs';
12
13
  /** Largest single file loaded, in bytes. */
13
14
  export declare const MAX_FILE_BYTES: number;
14
15
  /** Most files considered per load. */
@@ -17,6 +18,24 @@ export declare const MAX_FILES = 2000;
17
18
  export declare const MAX_DEPTH = 6;
18
19
  /** Total bytes of text loaded per call. */
19
20
  export declare const MAX_TOTAL_BYTES: number;
21
+ /** Discovery stops here so a hostile tree cannot make the walk itself unbounded. */
22
+ export declare const MAX_DISCOVERED_FILES: number;
23
+ /**
24
+ * Most directory entries one external source may examine in total, across every
25
+ * directory and every root of the source. The discovery ceiling counts accepted
26
+ * candidates only, so this bounds the work a tree of non-matching entries can
27
+ * cost.
28
+ */
29
+ export declare const MAX_LISTED_ENTRIES: number;
30
+ /** What a walk skips and what it indexes; the part of a kind a source shares. */
31
+ interface WalkSpec {
32
+ /** Lower-case file names that are never indexed. */
33
+ readonly excludedFiles: readonly string[];
34
+ /** Directory names that are never descended into. */
35
+ readonly excludedDirs: readonly string[];
36
+ /** Lower-case file extensions that are indexed, dot included. Defaults to `.md`. */
37
+ readonly extensions?: readonly string[];
38
+ }
20
39
  /**
21
40
  * One entry per document kind. Every Markdown file under a kind's directory
22
41
  * counts, except a README and anything under `_archived/`. The loader reads the
@@ -91,6 +110,42 @@ export interface CorpusLoad {
91
110
  /** Bounded, sanitised, in deterministic order. */
92
111
  readonly warnings: readonly string[];
93
112
  }
113
+ /** Strip terminal escapes and control characters, then cap the length. */
114
+ export declare function cleanForWarning(text: string): string;
115
+ export declare function compareCodeUnits(a: string, b: string): number;
116
+ export declare function safeLstat(target: string): fs.Stats | null;
117
+ export declare function safeReaddir(target: string): string[] | null;
118
+ /**
119
+ * The entries one external source may still examine. One budget is shared by
120
+ * every directory and root of the source; the built-in kinds have none and list
121
+ * a directory whole.
122
+ */
123
+ export interface EntryBudget {
124
+ remaining: number;
125
+ /** True once a listing was cut short or skipped for want of budget. */
126
+ exhausted: boolean;
127
+ }
128
+ export declare function newEntryBudget(limit?: number): EntryBudget;
129
+ /**
130
+ * List a directory against a budget: at most `budget.remaining` names, which
131
+ * are then taken off the budget. A directory with more entries, or one reached
132
+ * after the budget ran out, marks the budget exhausted; a spent budget opens
133
+ * nothing. `null` when the directory cannot be read.
134
+ */
135
+ export declare function listDirectoryWithin(target: string, budget: EntryBudget): BoundedListing | null;
136
+ export interface BoundedListing {
137
+ readonly names: string[];
138
+ /** True when the directory held more than `limit` entries, so `names` is a subset. */
139
+ readonly truncated: boolean;
140
+ }
141
+ /**
142
+ * List a directory, holding at most `limit` names. Entries stream from the
143
+ * directory handle, so a folder of millions of entries is never materialized.
144
+ * A listing cut at `limit` is a subset in directory order; the caller sorts
145
+ * what it got and says the listing was cut. `null` when the directory cannot
146
+ * be opened or read.
147
+ */
148
+ export declare function listDirectoryBounded(target: string, limit: number): BoundedListing | null;
94
149
  /**
95
150
  * The project root: the nearest ancestor of `startDir` holding `.git` (a
96
151
  * directory, or a file for a worktree), resolved with `realpath`; otherwise
@@ -103,6 +158,42 @@ export interface CorpusLoad {
103
158
  * the standalone search bin. This is the second use; extract at the third.
104
159
  */
105
160
  export declare function resolveSearchRoot(startDir: string): string;
161
+ export declare function isInside(root: string, candidate: string): boolean;
162
+ export type SymlinkedComponent = 'root' | 'ancestor';
163
+ /**
164
+ * Whether a component of the path from `container` down to `target` is a
165
+ * symlink, checked with `lstat` one component at a time: `root` when the final
166
+ * component is, `ancestor` when one above it is. A `target` that is not lexically
167
+ * inside `container` has no components to check; the realpath containment check
168
+ * owns it. A component that is missing or cannot be inspected ends the check.
169
+ */
170
+ export declare function symlinkedComponent(container: string, target: string): SymlinkedComponent | undefined;
171
+ export interface SourceFileCandidate {
172
+ /** Path relative to the walked directory, with `/` separators. */
173
+ readonly relative: string;
174
+ readonly absolute: string;
175
+ readonly size: number;
176
+ }
177
+ export declare function narrowString(value: unknown): string | undefined;
178
+ /** A string with something in it after trimming; anything else is `undefined`. */
179
+ export declare function narrowNonEmptyString(value: unknown): string | undefined;
180
+ export type ParsedMarkdown = {
181
+ readonly ok: true;
182
+ readonly data: Record<string, unknown>;
183
+ /** Text after the frontmatter block, or the whole file when there is none. */
184
+ readonly body: string;
185
+ } | {
186
+ readonly ok: false;
187
+ readonly reason: string;
188
+ };
189
+ /**
190
+ * The frontmatter step both the built-in kinds and the external Markdown
191
+ * sources share: strips a leading BOM, supplies the newline `parseFrontmatter`'s
192
+ * regex needs when the closing `---` is the last line, and rejects invalid YAML
193
+ * and a block that is not a mapping. No frontmatter is a body-only file with
194
+ * empty data, with no warning.
195
+ */
196
+ export declare function parseMarkdownFrontmatter(content: string): ParsedMarkdown;
106
197
  export type ParsedDocument = {
107
198
  readonly ok: true;
108
199
  readonly document: CorpusDocument;
@@ -111,16 +202,118 @@ export type ParsedDocument = {
111
202
  readonly reason: string;
112
203
  };
113
204
  /**
114
- * Parse one file. Wraps `parseFrontmatter`: strips a leading BOM, supplies the
115
- * newline its regex needs when the closing `---` is the last line, and reads
116
- * only known fields, narrowing each. No frontmatter means a body-only document
117
- * with no warning. Invalid YAML, or a frontmatter block that is not a mapping,
118
- * is not indexable.
205
+ * Parse one file. Wraps `parseMarkdownFrontmatter` and reads only known
206
+ * fields, narrowing each. No frontmatter means a body-only document with no
207
+ * warning. Invalid YAML, or a frontmatter block that is not a mapping, is not
208
+ * indexable.
119
209
  */
120
210
  export declare function parseCorpusDocument(kind: DocumentKind, relativePath: string, content: string): ParsedDocument;
211
+ export type ParsedSourceFile<T> = {
212
+ readonly ok: true;
213
+ readonly value: T;
214
+ } | {
215
+ readonly ok: false;
216
+ readonly reason: string;
217
+ };
218
+ export interface DiscoverSourceFilesOptions {
219
+ /** The directory to walk. Not yet resolved: this function takes its realpath. */
220
+ readonly directory: string;
221
+ /**
222
+ * When set, the walked directory's realpath must lie strictly inside this
223
+ * directory's realpath, or nothing is loaded. Built-in kinds and project
224
+ * roots set it; user roots do not.
225
+ */
226
+ readonly containedIn?: string | undefined;
227
+ readonly walk: WalkSpec;
228
+ /**
229
+ * An external source's entry budget, shared with the other roots of the same
230
+ * source. Left out, every directory is listed whole.
231
+ */
232
+ readonly entryBudget?: EntryBudget | undefined;
233
+ /** Names the directory in warnings: `spec.dir` for a built-in kind, the source name for an external one. */
234
+ readonly label: string;
235
+ }
236
+ export type SourceDiscovery = {
237
+ /**
238
+ * `missing`: nothing is there (or it is not a directory). `unreadable`:
239
+ * an access or I/O failure. `outside`: the realpath left `containedIn`.
240
+ */
241
+ readonly state: 'missing' | 'unreadable' | 'outside';
242
+ } | {
243
+ readonly state: 'found';
244
+ /** Sorted by relative path in code-unit order; every file the walk found, before any cap. */
245
+ readonly candidates: SourceFileCandidate[];
246
+ /** True when discovery stopped at its ceiling, so counts are lower bounds. */
247
+ readonly truncated: boolean;
248
+ };
249
+ /**
250
+ * Probe one directory and walk it without reading any file. Symlinks are
251
+ * skipped at every depth. Absence is a quiet `missing`; any other failure is
252
+ * `unreadable` and says so once, in fixed text with no error detail. Never
253
+ * throws.
254
+ */
255
+ export declare function discoverSourceFiles(options: DiscoverSourceFilesOptions, warnings: string[]): SourceDiscovery;
256
+ export interface ReadSourceFilesOptions<T> {
257
+ /** Prepended to a candidate's relative path to form the path a record carries. */
258
+ readonly displayPrefix: string;
259
+ /** Names the source in the file-limit and text-budget warnings. */
260
+ readonly label: string;
261
+ /** Largest single file read, in bytes. */
262
+ readonly maxFileBytes: number;
263
+ /**
264
+ * Bytes of a read budget shared with other sources that this call may still
265
+ * use. Left out for the built-in kinds, which share none.
266
+ */
267
+ readonly sharedBytes?: number | undefined;
268
+ /** Parses one file; `displayPath` is `displayPrefix/relative`. */
269
+ readonly parse: (displayPath: string, content: string) => ParsedSourceFile<T>;
270
+ }
271
+ export interface SourceFilesRead<T> {
272
+ readonly items: T[];
273
+ /** Stat bytes of the files opened, the figure a shared read budget draws on. */
274
+ readonly bytesRead: number;
275
+ }
276
+ /** What bounds a read: the per-file cap, the per-source text budget or the budget shared with other sources. */
277
+ export type ReadBound = 'file' | 'source' | 'shared';
278
+ export interface ReadAllowance {
279
+ /** The most a file may hold: a read stops at one byte past it. */
280
+ readonly limit: number;
281
+ readonly bound: ReadBound;
282
+ }
283
+ /**
284
+ * How much the next file may hold, given the bytes this source has already
285
+ * used: the smallest of the per-file cap and what is left of the per-source
286
+ * text budget and of the shared budget. The size seen at discovery is not part
287
+ * of it, because a file can grow after discovery.
288
+ */
289
+ export declare function readAllowance(used: number, maxFileBytes: number, sharedBytes?: number): ReadAllowance;
290
+ /**
291
+ * Read the first `MAX_FILES` of `candidates` (already sorted), within the
292
+ * per-file cap and the per-call `MAX_TOTAL_BYTES`, and warn about whatever the
293
+ * file limit or the text budget left out. `truncated` is the discovery flag.
294
+ * Never throws.
295
+ */
296
+ export declare function readSourceFiles<T>(options: ReadSourceFilesOptions<T>, candidates: readonly SourceFileCandidate[], truncated: boolean, warnings: string[]): SourceFilesRead<T>;
297
+ export type SourceFilesLoad<T> = {
298
+ readonly state: 'missing' | 'unreadable' | 'outside';
299
+ readonly warnings: string[];
300
+ } | {
301
+ readonly state: 'loaded';
302
+ readonly items: T[];
303
+ readonly bytesRead: number;
304
+ readonly warnings: string[];
305
+ };
306
+ /**
307
+ * The shared walk-and-read core: discover one directory, then read what the
308
+ * caps allow. `loadCorpus` is a caller of it for the built-in kinds; the
309
+ * external loader uses `discoverSourceFiles` and `readSourceFiles` directly so
310
+ * it can merge several roots and check a shared budget before reading.
311
+ */
312
+ export declare function loadSourceFiles<T>(discover: DiscoverSourceFilesOptions, read: ReadSourceFilesOptions<T>): SourceFilesLoad<T>;
121
313
  /**
122
314
  * Load every indexable document of one kind under `root`, sorted by
123
315
  * root-relative path in code-unit order. `root` should come from
124
316
  * `resolveSearchRoot`.
125
317
  */
126
318
  export declare function loadCorpus(root: string, kind?: DocumentKind): CorpusLoad;
319
+ export {};
@@ -0,0 +1,59 @@
1
+ /**
2
+ * The Deciduous reader behind `format: "deciduous"` external sources: turns a
3
+ * sync folder (`nodes/<change_id>.json` and `edges/<edge_id>.json`) into
4
+ * decision-shaped records with edge rationale as searchable text.
5
+ *
6
+ * It has its own scan instead of the shared `min(candidates, MAX_FILES)` cut:
7
+ * every root is probed first, then `nodes/` of every root in root order and
8
+ * then `edges/` of every root, in that explicit order and not a path sort.
9
+ * `MAX_DISCOVERED_FILES` bounds the files scanned, `MAX_LISTED_ENTRIES` bounds
10
+ * the entries one listing examines, `MAX_FILES` bounds the kept records, and
11
+ * the byte caps still apply, so edge files and filtered nodes never use up the
12
+ * record cap.
13
+ *
14
+ * Trust. File contents are untrusted. A file is opened only by a path built
15
+ * from a directory listing whose file stem passed the id charset, never from an
16
+ * id read out of content, and a display path is built only from that stem.
17
+ * Content ids (`change_id`, edge endpoints) are keys in maps and sets and
18
+ * nothing else. Symlinks are skipped at every depth. Reads files only.
19
+ *
20
+ * Import rule: node builtins, `search-bounded-read.ts` and `search-corpus.ts`
21
+ * only. It never imports the dispatcher (`search-external.ts`), which adapts
22
+ * its result.
23
+ */
24
+ export type DeciduousState = 'loaded' | 'empty' | 'missing' | 'unreadable' | 'dropped';
25
+ /** A record as the dispatcher receives it; the dispatcher adds the source and origin. */
26
+ export interface DeciduousRecord {
27
+ /** `<source>/nodes/<id>.json`, built from a validated file stem. */
28
+ readonly path: string;
29
+ readonly title: string;
30
+ readonly nodeType: string;
31
+ readonly body: string;
32
+ }
33
+ export interface DeciduousLoad {
34
+ readonly state: DeciduousState;
35
+ readonly records: readonly DeciduousRecord[];
36
+ /** Bounded, sanitised, in deterministic order. They name the source, never a file or link target. */
37
+ readonly warnings: readonly string[];
38
+ /** Stat bytes of the files opened, the figure a shared read budget draws on. */
39
+ readonly bytesUsed: number;
40
+ }
41
+ export interface DeciduousLoadInput {
42
+ readonly name: string;
43
+ readonly roots: readonly string[];
44
+ /** The checkout root a project source must stay inside; unset for a user source. */
45
+ readonly containedIn?: string | undefined;
46
+ /** Bytes the source may open; a source that needs more is dropped whole, unread. */
47
+ readonly remainingBytes: number;
48
+ /** Largest single file read, in bytes. */
49
+ readonly maxFileBytes: number;
50
+ /** Directory entries the source may examine in total. Defaults to `MAX_LISTED_ENTRIES`. */
51
+ readonly entryLimit?: number | undefined;
52
+ }
53
+ /**
54
+ * Load one Deciduous source: scan `nodes/` then `edges/` of every root, read
55
+ * the nodes, keep the decision-shaped live ones, then fold edge rationale into
56
+ * the kept nodes. Never throws: a failure becomes a state and a bounded
57
+ * warning.
58
+ */
59
+ export declare function loadDeciduousRecords(input: DeciduousLoadInput): DeciduousLoad;