@fro.bot/systematic 3.29.1 → 3.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HARNESSES.md +10 -5
- package/agents/research/learnings-researcher.md +30 -62
- package/dist/claude-code-search.d.ts +8 -4
- package/dist/cli.d.ts +18 -7
- package/dist/cli.js +47 -23
- package/dist/{index-e467g4b6.js → index-10dym2d9.js} +2000 -211
- package/dist/index.js +8 -5
- package/dist/lib/config-protected-fields.d.ts +1 -1
- package/dist/lib/config-schema.d.ts +20 -0
- package/dist/lib/config.d.ts +8 -5
- package/dist/lib/search-bounded-read.d.ts +38 -0
- package/dist/lib/search-cli.d.ts +14 -1
- package/dist/lib/search-core.d.ts +79 -4
- package/dist/lib/search-corpus.d.ts +198 -5
- package/dist/lib/search-external-deciduous.d.ts +59 -0
- package/dist/lib/search-external-sources.d.ts +92 -0
- package/dist/lib/search-external.d.ts +75 -0
- package/dist/lib/search-project-identity.d.ts +42 -0
- package/dist/lib/search-render.d.ts +94 -1
- package/dist/lib/search-tool.d.ts +5 -0
- package/dist/lib/search-user-config.d.ts +116 -29
- package/dist/pi.js +4611 -2755
- package/package.json +7 -6
package/dist/index.js
CHANGED
|
@@ -20,12 +20,13 @@ import {
|
|
|
20
20
|
discoverSkills,
|
|
21
21
|
SEARCH_TOOL_NAME,
|
|
22
22
|
SEARCH_ARGS,
|
|
23
|
+
KIND_SELECTOR_EXTERNAL,
|
|
23
24
|
SEARCH_TOOL_DESCRIPTION,
|
|
24
25
|
SEARCH_ARG_DESCRIPTIONS,
|
|
25
26
|
SEARCH_NOTICES,
|
|
26
27
|
SEARCH_CONFIG_UNREADABLE_NOTE,
|
|
27
28
|
runSearch
|
|
28
|
-
} from "./index-
|
|
29
|
+
} from "./index-10dym2d9.js";
|
|
29
30
|
|
|
30
31
|
// src/index.ts
|
|
31
32
|
import path8 from "path";
|
|
@@ -982,11 +983,12 @@ function normalizePath(path) {
|
|
|
982
983
|
function isSearchEnabled(settings) {
|
|
983
984
|
return settings?.enabled !== false;
|
|
984
985
|
}
|
|
985
|
-
async function executeSearch(args, startDir, settings, unreadable) {
|
|
986
|
+
async function executeSearch(args, startDir, settings, unreadable, configOptions) {
|
|
986
987
|
try {
|
|
987
988
|
if (startDir === undefined)
|
|
988
989
|
return SEARCH_NOTICES.noProject;
|
|
989
990
|
const record = isRecord(args) ? args : {};
|
|
991
|
+
const external = record[SEARCH_ARGS.kind] === KIND_SELECTOR_EXTERNAL;
|
|
990
992
|
const response = await runSearch({
|
|
991
993
|
query: record[SEARCH_ARGS.query],
|
|
992
994
|
kind: record[SEARCH_ARGS.kind],
|
|
@@ -994,7 +996,8 @@ async function executeSearch(args, startDir, settings, unreadable) {
|
|
|
994
996
|
}, {
|
|
995
997
|
startDir,
|
|
996
998
|
settings,
|
|
997
|
-
|
|
999
|
+
configOptions,
|
|
1000
|
+
...unreadable === true && !external && { notes: [SEARCH_CONFIG_UNREADABLE_NOTE] }
|
|
998
1001
|
});
|
|
999
1002
|
return response.text;
|
|
1000
1003
|
} catch {
|
|
@@ -1005,7 +1008,7 @@ function usableDirectory(value) {
|
|
|
1005
1008
|
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
1006
1009
|
}
|
|
1007
1010
|
function createSearchTool(options) {
|
|
1008
|
-
const { settings, unreadable } = options;
|
|
1011
|
+
const { settings, unreadable, configOptions } = options;
|
|
1009
1012
|
return {
|
|
1010
1013
|
description: SEARCH_TOOL_DESCRIPTION,
|
|
1011
1014
|
args: {
|
|
@@ -1014,7 +1017,7 @@ function createSearchTool(options) {
|
|
|
1014
1017
|
[SEARCH_ARGS.limit]: number().int().optional().describe(SEARCH_ARG_DESCRIPTIONS.limit)
|
|
1015
1018
|
},
|
|
1016
1019
|
async execute(args, context) {
|
|
1017
|
-
return executeSearch(args, usableDirectory(context.directory), settings, unreadable);
|
|
1020
|
+
return executeSearch(args, usableDirectory(context.directory), settings, unreadable, configOptions);
|
|
1018
1021
|
}
|
|
1019
1022
|
};
|
|
1020
1023
|
}
|
|
@@ -18,5 +18,5 @@
|
|
|
18
18
|
* reason it exists as a separate file rather than living directly in
|
|
19
19
|
* `config.ts`.
|
|
20
20
|
*/
|
|
21
|
-
export declare const CONFIG_PROTECTED_FIELD_PATHS: readonly ['profiles', 'allow_project_profiles', 'search', 'agents.*.model', 'agents.*.permission', 'agents.*.skills', 'agents.*.variant', 'agents.*.opencode', 'agents.*.pi', 'categories.*.model', 'categories.*.permission', 'categories.*.skills', 'categories.*.variant', 'categories.*.opencode', 'categories.*.pi'];
|
|
21
|
+
export declare const CONFIG_PROTECTED_FIELD_PATHS: readonly ['profiles', 'allow_project_profiles', 'search.enabled', 'search.allow_project_sources', 'agents.*.model', 'agents.*.permission', 'agents.*.skills', 'agents.*.variant', 'agents.*.opencode', 'agents.*.pi', 'categories.*.model', 'categories.*.permission', 'categories.*.skills', 'categories.*.variant', 'categories.*.opencode', 'categories.*.pi'];
|
|
22
22
|
export type ConfigProtectedFieldPath = (typeof CONFIG_PROTECTED_FIELD_PATHS)[number];
|
|
@@ -387,6 +387,16 @@ export declare function createSystematicConfigSchema(opts: SystematicConfigSchem
|
|
|
387
387
|
allow_project_profiles: z.ZodDefault<z.ZodBoolean>;
|
|
388
388
|
search: z.ZodOptional<z.ZodObject<{
|
|
389
389
|
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
390
|
+
allow_project_sources: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
391
|
+
sources: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
392
|
+
name: z.ZodString;
|
|
393
|
+
format: z.ZodEnum<{
|
|
394
|
+
deciduous: "deciduous";
|
|
395
|
+
markdown: "markdown";
|
|
396
|
+
}>;
|
|
397
|
+
roots: z.ZodArray<z.ZodString>;
|
|
398
|
+
projects: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
399
|
+
}, z.core.$strict>>>;
|
|
390
400
|
}, z.core.$strict>>;
|
|
391
401
|
disabled_skills: z.ZodDefault<z.ZodArray<z.ZodEnum<{
|
|
392
402
|
[x: string]: string;
|
|
@@ -585,6 +595,16 @@ export declare const SystematicConfigSchema: z.ZodObject<{
|
|
|
585
595
|
allow_project_profiles: z.ZodDefault<z.ZodBoolean>;
|
|
586
596
|
search: z.ZodOptional<z.ZodObject<{
|
|
587
597
|
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
598
|
+
allow_project_sources: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
599
|
+
sources: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
600
|
+
name: z.ZodString;
|
|
601
|
+
format: z.ZodEnum<{
|
|
602
|
+
deciduous: "deciduous";
|
|
603
|
+
markdown: "markdown";
|
|
604
|
+
}>;
|
|
605
|
+
roots: z.ZodArray<z.ZodString>;
|
|
606
|
+
projects: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
607
|
+
}, z.core.$strict>>>;
|
|
588
608
|
}, z.core.$strict>>;
|
|
589
609
|
disabled_skills: z.ZodDefault<z.ZodArray<z.ZodEnum<{
|
|
590
610
|
[x: string]: string;
|
package/dist/lib/config.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CONFIG_PROTECTED_FIELD_PATHS, type ConfigProtectedFieldPath } from './config-protected-fields.js';
|
|
2
2
|
import { type RoutingTarget } from './routing-resolver.js';
|
|
3
|
-
import type
|
|
3
|
+
import { type SearchSchemaOutput } from './search-user-config.js';
|
|
4
4
|
export interface BootstrapConfig {
|
|
5
5
|
enabled: boolean;
|
|
6
6
|
file?: string;
|
|
@@ -174,11 +174,14 @@ export interface SystematicConfig {
|
|
|
174
174
|
skills_as_commands: boolean;
|
|
175
175
|
allow_project_profiles: boolean;
|
|
176
176
|
/**
|
|
177
|
-
* Merged `search` settings from user and custom config only
|
|
178
|
-
*
|
|
179
|
-
*
|
|
177
|
+
* Merged `search` settings from user and custom config only: `enabled`
|
|
178
|
+
* (last-defining), `allow_project_sources` (union) and the user `sources`
|
|
179
|
+
* (by name, custom winning). Absent when neither defines one: there is no
|
|
180
|
+
* default, and the core applies its own at use time. A project config's
|
|
181
|
+
* `search.sources` are validated at load and then discarded; they never
|
|
182
|
+
* appear here. The per-call external reader is their only consumer.
|
|
180
183
|
*/
|
|
181
|
-
search?:
|
|
184
|
+
search?: SearchSchemaOutput;
|
|
182
185
|
/**
|
|
183
186
|
* Set (to `true`) when a user or custom `search` block, or its file, could
|
|
184
187
|
* not be read. That block is ignored and search results carry a
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one bounded file read behind every search reader that opens a file whose
|
|
3
|
+
* content an untrusted checkout or folder controls: the project config, the
|
|
4
|
+
* project identity files and the corpus and external source files.
|
|
5
|
+
*
|
|
6
|
+
* Import rule: node builtins only. The `systematic-search` bin bundles it, so
|
|
7
|
+
* it must never import `config.ts`, `config-schema.ts` or any module that does.
|
|
8
|
+
*/
|
|
9
|
+
export type BoundedRead = {
|
|
10
|
+
readonly kind: 'bytes';
|
|
11
|
+
readonly bytes: Buffer;
|
|
12
|
+
}
|
|
13
|
+
/** The path does not exist (`ENOENT`). */
|
|
14
|
+
| {
|
|
15
|
+
readonly kind: 'absent';
|
|
16
|
+
}
|
|
17
|
+
/** The opened descriptor is not a regular file, such as a directory or a FIFO. */
|
|
18
|
+
| {
|
|
19
|
+
readonly kind: 'not-regular';
|
|
20
|
+
}
|
|
21
|
+
/** The content reached `maxBytes + 1`; `bytesRead` is what was actually read. */
|
|
22
|
+
| {
|
|
23
|
+
readonly kind: 'too-large';
|
|
24
|
+
readonly bytesRead: number;
|
|
25
|
+
}
|
|
26
|
+
/** Any other open or read failure. */
|
|
27
|
+
| {
|
|
28
|
+
readonly kind: 'failed';
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Read a file through an open fd, never more than `maxBytes + 1` bytes.
|
|
32
|
+
* Reaching `maxBytes + 1` rejects the file, so a file that is already too
|
|
33
|
+
* large and one that grows past the cap while it is being read fail the same
|
|
34
|
+
* way, and the size is never trusted from a `stat`. The open is non-blocking
|
|
35
|
+
* and the fd must be a regular file, so a FIFO or device planted at the path
|
|
36
|
+
* cannot hang the reader. Never throws.
|
|
37
|
+
*/
|
|
38
|
+
export declare function readBytesBounded(filePath: string, maxBytes: number): BoundedRead;
|
package/dist/lib/search-cli.d.ts
CHANGED
|
@@ -13,9 +13,10 @@
|
|
|
13
13
|
* failure; 2 for usage errors. There is no `--json`.
|
|
14
14
|
*
|
|
15
15
|
* Import rule: this module is bundled into the `systematic-search` bin, so it
|
|
16
|
-
* may import only `search-core.ts
|
|
16
|
+
* may import only `search-core.ts`, node builtins and types.
|
|
17
17
|
*/
|
|
18
18
|
import { type SearchSettings } from './search-core.js';
|
|
19
|
+
import type { UserLevelConfigDirOptions } from './search-user-config.js';
|
|
19
20
|
/**
|
|
20
21
|
* The usage text for one surface. `commandName` is what that surface's users
|
|
21
22
|
* type (`systematic search` or `systematic-search`), so the text never tells a
|
|
@@ -24,6 +25,12 @@ import { type SearchSettings } from './search-core.js';
|
|
|
24
25
|
export declare function searchCliUsage(commandName: string): string;
|
|
25
26
|
/** What `search.enabled: false` prints. The researcher falls back to grep on it. */
|
|
26
27
|
export declare const SEARCH_CLI_DISABLED_NOTICE = "Search is disabled (search.enabled is false in your Systematic config), so there is nothing to search.";
|
|
28
|
+
/**
|
|
29
|
+
* Printed once to stderr when `--kind external` finds the project's config
|
|
30
|
+
* file structurally broken. Fixed text: no path, no value. The reply
|
|
31
|
+
* carries the aggregate line and the bounded warnings.
|
|
32
|
+
*/
|
|
33
|
+
export declare const SEARCH_CLI_PROJECT_CONFIG_WARNING = "[systematic] The project Systematic config could not be read, so it proposes no external sources.";
|
|
27
34
|
/** What a settings loader returns. The bin's user-level reader returns the same shape. */
|
|
28
35
|
export interface SearchCliSettingsLoad {
|
|
29
36
|
readonly settings?: SearchSettings | undefined;
|
|
@@ -41,6 +48,12 @@ export interface SearchCliOptions {
|
|
|
41
48
|
/** The command the usage text names: `systematic search` or `systematic-search`. */
|
|
42
49
|
readonly usageName: string;
|
|
43
50
|
readonly loadSettings: SearchCliSettingsLoader;
|
|
51
|
+
/**
|
|
52
|
+
* The user-level config directories `--kind external` reads its allow list
|
|
53
|
+
* and user sources from. The caller passes the same options it gives its
|
|
54
|
+
* settings loader, so one override reaches both.
|
|
55
|
+
*/
|
|
56
|
+
readonly configOptions?: UserLevelConfigDirOptions | undefined;
|
|
44
57
|
readonly outputSink?: (message: string) => void;
|
|
45
58
|
readonly errorSink?: (message: string) => void;
|
|
46
59
|
}
|
|
@@ -8,6 +8,12 @@
|
|
|
8
8
|
* I/O: the start directory, the settings, and mapping the response to its
|
|
9
9
|
* host's return shape. Adapters never format text themselves.
|
|
10
10
|
*
|
|
11
|
+
* `kind: "external"` is a selector beside the four kinds, not a fifth kind. It
|
|
12
|
+
* takes its own branch: the core resolves the active sources for the call
|
|
13
|
+
* through an injectable resolver, loads each one, ranks each source on its own
|
|
14
|
+
* index, and hands the sections to `renderExternalResult`. Built-in kinds never
|
|
15
|
+
* reach that branch, so their output is unchanged.
|
|
16
|
+
*
|
|
11
17
|
* This module also owns the `SearchSettings` type, the tool name and argument
|
|
12
18
|
* constants, and the lifecycle constants and resolver. The resolver follows
|
|
13
19
|
* the learnings-researcher's rules: at most 5 hops, a visited set, and a form
|
|
@@ -18,8 +24,11 @@
|
|
|
18
24
|
* `Caution: successor unresolved`. Nothing is dropped.
|
|
19
25
|
*/
|
|
20
26
|
import { type CorpusDocument, type CorpusLoad, type DocumentKind } from './search-corpus.js';
|
|
27
|
+
import { type SourceLoad } from './search-external.js';
|
|
28
|
+
import { type ActiveSource, type ExternalSourcesInput, type ExternalSourcesResolution } from './search-external-sources.js';
|
|
21
29
|
import { type Bm25fParams } from './search-lexical.js';
|
|
22
|
-
import { type SearchHitStatus, type SearchResult, SUCCESSOR_LABEL, SUPERSEDES_LABEL } from './search-render.js';
|
|
30
|
+
import { type ExternalFixedReply, type ExternalSearchResult, type ExternalSection, type SearchHitStatus, type SearchResult, SUCCESSOR_LABEL, SUPERSEDES_LABEL } from './search-render.js';
|
|
31
|
+
import { type UserLevelConfigDirOptions } from './search-user-config.js';
|
|
23
32
|
export type { SearchHit, SearchResult } from './search-render.js';
|
|
24
33
|
export { SUCCESSOR_LABEL, SUPERSEDES_LABEL };
|
|
25
34
|
/** Tool name shared by every host surface. */
|
|
@@ -36,16 +45,24 @@ export declare const MAX_LIMIT = 20;
|
|
|
36
45
|
export declare const MAX_QUERY_CHARS = 2000;
|
|
37
46
|
/** The `kind` value that searches every kind in one ranking. It is a selector, not a table key. */
|
|
38
47
|
export declare const KIND_SELECTOR_ALL = "all";
|
|
48
|
+
/**
|
|
49
|
+
* The `kind` value that searches the user-allowed external sources. Like `all`
|
|
50
|
+
* it is a selector, not a table key, and it is exclusive: it does not combine
|
|
51
|
+
* with a kind or with `all`.
|
|
52
|
+
*/
|
|
53
|
+
export declare const KIND_SELECTOR_EXTERNAL = "external";
|
|
54
|
+
/** Most hits one external source returns, whatever `limit` asks for. */
|
|
55
|
+
export declare const MAX_EXTERNAL_HITS_PER_SOURCE = 5;
|
|
39
56
|
/**
|
|
40
57
|
* The tool text every host surface shows the model. `limit` is a plain integer
|
|
41
58
|
* in every host schema, so its range is stated here and nowhere else: the core
|
|
42
59
|
* owns the out-of-range error and a host never pre-empts it.
|
|
43
60
|
*/
|
|
44
|
-
export declare const SEARCH_TOOL_DESCRIPTION = "Search this project's durable outputs by symptom, error text, module or topic: the docs/solutions learnings, and the docs/plans, docs/brainstorms and docs/ideation documents that earlier work wrote. Returns ranked hits with path, title, kind, snippet, tags and status. A superseded document is replaced by its successor in the same kind, a successor in another kind is named on a Successor line, and a stale one carries a caution. Read a hit's path for the full text. Use it before debugging a failure or planning work that a past learning, plan, brainstorm or ideation doc may already cover.";
|
|
61
|
+
export declare const SEARCH_TOOL_DESCRIPTION = "Search this project's durable outputs by symptom, error text, module or topic: the docs/solutions learnings, and the docs/plans, docs/brainstorms and docs/ideation documents that earlier work wrote. Returns ranked hits with path, title, kind, snippet, tags and status. A superseded document is replaced by its successor in the same kind, a successor in another kind is named on a Successor line, and a stale one carries a caution. Read a hit's path for the full text. Use it before debugging a failure or planning work that a past learning, plan, brainstorm or ideation doc may already cover. With kind external, a hit's path is `<source>/<path>`: the source name, then a path inside one of that source's roots, which the source's section header lists. Strip the source name to resolve it. The reply does not say which root a hit came from; with several roots, the file is under the first listed root that has it.";
|
|
45
62
|
export declare const SEARCH_ARG_DESCRIPTIONS: {
|
|
46
63
|
readonly query: "What to look for: a symptom, error message, module or topic, in plain words. At most 2000 characters.";
|
|
47
|
-
readonly kind: `Optional. The kind of durable output to search: ${string}, or "all" for every kind in one ranking. Defaults to "solution".`;
|
|
48
|
-
readonly limit: "Optional. How many hits to return, an integer from 1 to 20. Defaults to 8.";
|
|
64
|
+
readonly kind: `Optional. The kind of durable output to search: ${string}, or "all" for every kind in one ranking, or "external" for the external decision sources the user allowed, each ranked on its own. Defaults to "solution".`;
|
|
65
|
+
readonly limit: "Optional. How many hits to return, an integer from 1 to 20. Defaults to 8. With kind \"external\" it applies to each source instead, and at most 5 hits per source are returned.";
|
|
49
66
|
};
|
|
50
67
|
/** One-line texts an adapter returns when it cannot reach the core. They carry no detail from the failure. */
|
|
51
68
|
export declare const SEARCH_NOTICES: {
|
|
@@ -86,11 +103,56 @@ export interface SearchOptions {
|
|
|
86
103
|
readonly lexicalParams?: Bm25fParams | undefined;
|
|
87
104
|
/** Replace the disk loader. Tests use this to control load order. */
|
|
88
105
|
readonly corpusLoader?: ((root: string, kind: DocumentKind) => CorpusLoad) | undefined;
|
|
106
|
+
/**
|
|
107
|
+
* The user-level config directories `kind: "external"` reads its allow list
|
|
108
|
+
* and user sources from. A host or CLI config-dir override goes here; left
|
|
109
|
+
* out, the reader uses the home directory and `OPENCODE_CONFIG_DIR`.
|
|
110
|
+
*/
|
|
111
|
+
readonly configOptions?: UserLevelConfigDirOptions | undefined;
|
|
112
|
+
/**
|
|
113
|
+
* Replace the per-call external reader. The default resolves the active
|
|
114
|
+
* sources from the user files, the project file and the project identity.
|
|
115
|
+
* Tests use this to control what a call sees.
|
|
116
|
+
*/
|
|
117
|
+
readonly externalResolver?: ExternalResolver | undefined;
|
|
89
118
|
}
|
|
119
|
+
/** Reads everything for one `external` call and returns the active sources. The default never throws. */
|
|
120
|
+
export type ExternalResolver = (input: ExternalSourcesInput) => ExternalSourcesResolution;
|
|
121
|
+
/** The default resolver: the per-call reader in `search-external-sources.ts`. */
|
|
122
|
+
export declare const defaultExternalResolver: ExternalResolver;
|
|
123
|
+
/**
|
|
124
|
+
* What an `external` call produced: one of the core's fixed replies, or the
|
|
125
|
+
* ranked sections of the active sources.
|
|
126
|
+
*/
|
|
127
|
+
export type ExternalOutcome = {
|
|
128
|
+
readonly kind: 'fixed';
|
|
129
|
+
readonly reply: ExternalFixedReply;
|
|
130
|
+
} | {
|
|
131
|
+
readonly kind: 'sections';
|
|
132
|
+
readonly result: ExternalSearchResult;
|
|
133
|
+
};
|
|
134
|
+
/**
|
|
135
|
+
* A response is tagged by `ok`, then by `selector`: a built-in response carries
|
|
136
|
+
* `result` and an external one carries `external`, and neither carries the
|
|
137
|
+
* other, so a consumer must narrow on `selector` before reading either. Every
|
|
138
|
+
* response carries `text`, which is all the adapters read.
|
|
139
|
+
*/
|
|
90
140
|
export type SearchResponse = {
|
|
91
141
|
readonly ok: true;
|
|
142
|
+
readonly selector: 'builtin';
|
|
92
143
|
readonly text: string;
|
|
93
144
|
readonly result: SearchResult;
|
|
145
|
+
} | {
|
|
146
|
+
readonly ok: true;
|
|
147
|
+
readonly selector: 'external';
|
|
148
|
+
readonly text: string;
|
|
149
|
+
readonly external: ExternalOutcome;
|
|
150
|
+
/**
|
|
151
|
+
* Present (and `true`) when the project's config file had a structural
|
|
152
|
+
* error, so it proposed nothing. The reply text already says so; the CLI
|
|
153
|
+
* and the bin also print one stderr warning for it.
|
|
154
|
+
*/
|
|
155
|
+
readonly projectConfigUnreadable?: true;
|
|
94
156
|
} | {
|
|
95
157
|
readonly ok: false;
|
|
96
158
|
readonly text: string;
|
|
@@ -144,3 +206,16 @@ export declare function lifecycleLines(entry: ResolvedEntry): string[];
|
|
|
144
206
|
* request ranks exactly as it does alone.
|
|
145
207
|
*/
|
|
146
208
|
export declare function runSearch(request: SearchRequest, options: SearchOptions): Promise<SearchResponse>;
|
|
209
|
+
interface ActiveLoad {
|
|
210
|
+
readonly source: ActiveSource;
|
|
211
|
+
readonly load: SourceLoad;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Rank every source on its own and build the sections, ordered by name in
|
|
215
|
+
* code-unit order (then origin). Scores are not comparable across sources, so
|
|
216
|
+
* each source gets its own lexical index: `title` to title, `node_type` to
|
|
217
|
+
* tags and `body` to body. Hits within a section come best first with ties by
|
|
218
|
+
* path, capped at `limit` and at `MAX_EXTERNAL_HITS_PER_SOURCE`. A source that
|
|
219
|
+
* loaded no record is a state-only section.
|
|
220
|
+
*/
|
|
221
|
+
export declare function rankExternalSources(entries: readonly ActiveLoad[], queryTerms: readonly string[], limit: number, params?: Bm25fParams): ExternalSection[];
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* carry the narrowed `status`, `superseded_by` and `stale_reason` that the
|
|
10
10
|
* search core's resolver reads.
|
|
11
11
|
*/
|
|
12
|
+
import fs from 'node:fs';
|
|
12
13
|
/** Largest single file loaded, in bytes. */
|
|
13
14
|
export declare const MAX_FILE_BYTES: number;
|
|
14
15
|
/** Most files considered per load. */
|
|
@@ -17,6 +18,24 @@ export declare const MAX_FILES = 2000;
|
|
|
17
18
|
export declare const MAX_DEPTH = 6;
|
|
18
19
|
/** Total bytes of text loaded per call. */
|
|
19
20
|
export declare const MAX_TOTAL_BYTES: number;
|
|
21
|
+
/** Discovery stops here so a hostile tree cannot make the walk itself unbounded. */
|
|
22
|
+
export declare const MAX_DISCOVERED_FILES: number;
|
|
23
|
+
/**
|
|
24
|
+
* Most directory entries one external source may examine in total, across every
|
|
25
|
+
* directory and every root of the source. The discovery ceiling counts accepted
|
|
26
|
+
* candidates only, so this bounds the work a tree of non-matching entries can
|
|
27
|
+
* cost.
|
|
28
|
+
*/
|
|
29
|
+
export declare const MAX_LISTED_ENTRIES: number;
|
|
30
|
+
/** What a walk skips and what it indexes; the part of a kind a source shares. */
|
|
31
|
+
interface WalkSpec {
|
|
32
|
+
/** Lower-case file names that are never indexed. */
|
|
33
|
+
readonly excludedFiles: readonly string[];
|
|
34
|
+
/** Directory names that are never descended into. */
|
|
35
|
+
readonly excludedDirs: readonly string[];
|
|
36
|
+
/** Lower-case file extensions that are indexed, dot included. Defaults to `.md`. */
|
|
37
|
+
readonly extensions?: readonly string[];
|
|
38
|
+
}
|
|
20
39
|
/**
|
|
21
40
|
* One entry per document kind. Every Markdown file under a kind's directory
|
|
22
41
|
* counts, except a README and anything under `_archived/`. The loader reads the
|
|
@@ -91,6 +110,42 @@ export interface CorpusLoad {
|
|
|
91
110
|
/** Bounded, sanitised, in deterministic order. */
|
|
92
111
|
readonly warnings: readonly string[];
|
|
93
112
|
}
|
|
113
|
+
/** Strip terminal escapes and control characters, then cap the length. */
|
|
114
|
+
export declare function cleanForWarning(text: string): string;
|
|
115
|
+
export declare function compareCodeUnits(a: string, b: string): number;
|
|
116
|
+
export declare function safeLstat(target: string): fs.Stats | null;
|
|
117
|
+
export declare function safeReaddir(target: string): string[] | null;
|
|
118
|
+
/**
|
|
119
|
+
* The entries one external source may still examine. One budget is shared by
|
|
120
|
+
* every directory and root of the source; the built-in kinds have none and list
|
|
121
|
+
* a directory whole.
|
|
122
|
+
*/
|
|
123
|
+
export interface EntryBudget {
|
|
124
|
+
remaining: number;
|
|
125
|
+
/** True once a listing was cut short or skipped for want of budget. */
|
|
126
|
+
exhausted: boolean;
|
|
127
|
+
}
|
|
128
|
+
export declare function newEntryBudget(limit?: number): EntryBudget;
|
|
129
|
+
/**
|
|
130
|
+
* List a directory against a budget: at most `budget.remaining` names, which
|
|
131
|
+
* are then taken off the budget. A directory with more entries, or one reached
|
|
132
|
+
* after the budget ran out, marks the budget exhausted; a spent budget opens
|
|
133
|
+
* nothing. `null` when the directory cannot be read.
|
|
134
|
+
*/
|
|
135
|
+
export declare function listDirectoryWithin(target: string, budget: EntryBudget): BoundedListing | null;
|
|
136
|
+
export interface BoundedListing {
|
|
137
|
+
readonly names: string[];
|
|
138
|
+
/** True when the directory held more than `limit` entries, so `names` is a subset. */
|
|
139
|
+
readonly truncated: boolean;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* List a directory, holding at most `limit` names. Entries stream from the
|
|
143
|
+
* directory handle, so a folder of millions of entries is never materialized.
|
|
144
|
+
* A listing cut at `limit` is a subset in directory order; the caller sorts
|
|
145
|
+
* what it got and says the listing was cut. `null` when the directory cannot
|
|
146
|
+
* be opened or read.
|
|
147
|
+
*/
|
|
148
|
+
export declare function listDirectoryBounded(target: string, limit: number): BoundedListing | null;
|
|
94
149
|
/**
|
|
95
150
|
* The project root: the nearest ancestor of `startDir` holding `.git` (a
|
|
96
151
|
* directory, or a file for a worktree), resolved with `realpath`; otherwise
|
|
@@ -103,6 +158,42 @@ export interface CorpusLoad {
|
|
|
103
158
|
* the standalone search bin. This is the second use; extract at the third.
|
|
104
159
|
*/
|
|
105
160
|
export declare function resolveSearchRoot(startDir: string): string;
|
|
161
|
+
export declare function isInside(root: string, candidate: string): boolean;
|
|
162
|
+
export type SymlinkedComponent = 'root' | 'ancestor';
|
|
163
|
+
/**
|
|
164
|
+
* Whether a component of the path from `container` down to `target` is a
|
|
165
|
+
* symlink, checked with `lstat` one component at a time: `root` when the final
|
|
166
|
+
* component is, `ancestor` when one above it is. A `target` that is not lexically
|
|
167
|
+
* inside `container` has no components to check; the realpath containment check
|
|
168
|
+
* owns it. A component that is missing or cannot be inspected ends the check.
|
|
169
|
+
*/
|
|
170
|
+
export declare function symlinkedComponent(container: string, target: string): SymlinkedComponent | undefined;
|
|
171
|
+
export interface SourceFileCandidate {
|
|
172
|
+
/** Path relative to the walked directory, with `/` separators. */
|
|
173
|
+
readonly relative: string;
|
|
174
|
+
readonly absolute: string;
|
|
175
|
+
readonly size: number;
|
|
176
|
+
}
|
|
177
|
+
export declare function narrowString(value: unknown): string | undefined;
|
|
178
|
+
/** A string with something in it after trimming; anything else is `undefined`. */
|
|
179
|
+
export declare function narrowNonEmptyString(value: unknown): string | undefined;
|
|
180
|
+
export type ParsedMarkdown = {
|
|
181
|
+
readonly ok: true;
|
|
182
|
+
readonly data: Record<string, unknown>;
|
|
183
|
+
/** Text after the frontmatter block, or the whole file when there is none. */
|
|
184
|
+
readonly body: string;
|
|
185
|
+
} | {
|
|
186
|
+
readonly ok: false;
|
|
187
|
+
readonly reason: string;
|
|
188
|
+
};
|
|
189
|
+
/**
|
|
190
|
+
* The frontmatter step both the built-in kinds and the external Markdown
|
|
191
|
+
* sources share: strips a leading BOM, supplies the newline `parseFrontmatter`'s
|
|
192
|
+
* regex needs when the closing `---` is the last line, and rejects invalid YAML
|
|
193
|
+
* and a block that is not a mapping. No frontmatter is a body-only file with
|
|
194
|
+
* empty data, with no warning.
|
|
195
|
+
*/
|
|
196
|
+
export declare function parseMarkdownFrontmatter(content: string): ParsedMarkdown;
|
|
106
197
|
export type ParsedDocument = {
|
|
107
198
|
readonly ok: true;
|
|
108
199
|
readonly document: CorpusDocument;
|
|
@@ -111,16 +202,118 @@ export type ParsedDocument = {
|
|
|
111
202
|
readonly reason: string;
|
|
112
203
|
};
|
|
113
204
|
/**
|
|
114
|
-
* Parse one file. Wraps `
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
* is not indexable.
|
|
205
|
+
* Parse one file. Wraps `parseMarkdownFrontmatter` and reads only known
|
|
206
|
+
* fields, narrowing each. No frontmatter means a body-only document with no
|
|
207
|
+
* warning. Invalid YAML, or a frontmatter block that is not a mapping, is not
|
|
208
|
+
* indexable.
|
|
119
209
|
*/
|
|
120
210
|
export declare function parseCorpusDocument(kind: DocumentKind, relativePath: string, content: string): ParsedDocument;
|
|
211
|
+
export type ParsedSourceFile<T> = {
|
|
212
|
+
readonly ok: true;
|
|
213
|
+
readonly value: T;
|
|
214
|
+
} | {
|
|
215
|
+
readonly ok: false;
|
|
216
|
+
readonly reason: string;
|
|
217
|
+
};
|
|
218
|
+
export interface DiscoverSourceFilesOptions {
|
|
219
|
+
/** The directory to walk. Not yet resolved: this function takes its realpath. */
|
|
220
|
+
readonly directory: string;
|
|
221
|
+
/**
|
|
222
|
+
* When set, the walked directory's realpath must lie strictly inside this
|
|
223
|
+
* directory's realpath, or nothing is loaded. Built-in kinds and project
|
|
224
|
+
* roots set it; user roots do not.
|
|
225
|
+
*/
|
|
226
|
+
readonly containedIn?: string | undefined;
|
|
227
|
+
readonly walk: WalkSpec;
|
|
228
|
+
/**
|
|
229
|
+
* An external source's entry budget, shared with the other roots of the same
|
|
230
|
+
* source. Left out, every directory is listed whole.
|
|
231
|
+
*/
|
|
232
|
+
readonly entryBudget?: EntryBudget | undefined;
|
|
233
|
+
/** Names the directory in warnings: `spec.dir` for a built-in kind, the source name for an external one. */
|
|
234
|
+
readonly label: string;
|
|
235
|
+
}
|
|
236
|
+
export type SourceDiscovery = {
|
|
237
|
+
/**
|
|
238
|
+
* `missing`: nothing is there (or it is not a directory). `unreadable`:
|
|
239
|
+
* an access or I/O failure. `outside`: the realpath left `containedIn`.
|
|
240
|
+
*/
|
|
241
|
+
readonly state: 'missing' | 'unreadable' | 'outside';
|
|
242
|
+
} | {
|
|
243
|
+
readonly state: 'found';
|
|
244
|
+
/** Sorted by relative path in code-unit order; every file the walk found, before any cap. */
|
|
245
|
+
readonly candidates: SourceFileCandidate[];
|
|
246
|
+
/** True when discovery stopped at its ceiling, so counts are lower bounds. */
|
|
247
|
+
readonly truncated: boolean;
|
|
248
|
+
};
|
|
249
|
+
/**
|
|
250
|
+
* Probe one directory and walk it without reading any file. Symlinks are
|
|
251
|
+
* skipped at every depth. Absence is a quiet `missing`; any other failure is
|
|
252
|
+
* `unreadable` and says so once, in fixed text with no error detail. Never
|
|
253
|
+
* throws.
|
|
254
|
+
*/
|
|
255
|
+
export declare function discoverSourceFiles(options: DiscoverSourceFilesOptions, warnings: string[]): SourceDiscovery;
|
|
256
|
+
export interface ReadSourceFilesOptions<T> {
|
|
257
|
+
/** Prepended to a candidate's relative path to form the path a record carries. */
|
|
258
|
+
readonly displayPrefix: string;
|
|
259
|
+
/** Names the source in the file-limit and text-budget warnings. */
|
|
260
|
+
readonly label: string;
|
|
261
|
+
/** Largest single file read, in bytes. */
|
|
262
|
+
readonly maxFileBytes: number;
|
|
263
|
+
/**
|
|
264
|
+
* Bytes of a read budget shared with other sources that this call may still
|
|
265
|
+
* use. Left out for the built-in kinds, which share none.
|
|
266
|
+
*/
|
|
267
|
+
readonly sharedBytes?: number | undefined;
|
|
268
|
+
/** Parses one file; `displayPath` is `displayPrefix/relative`. */
|
|
269
|
+
readonly parse: (displayPath: string, content: string) => ParsedSourceFile<T>;
|
|
270
|
+
}
|
|
271
|
+
export interface SourceFilesRead<T> {
|
|
272
|
+
readonly items: T[];
|
|
273
|
+
/** Stat bytes of the files opened, the figure a shared read budget draws on. */
|
|
274
|
+
readonly bytesRead: number;
|
|
275
|
+
}
|
|
276
|
+
/** What bounds a read: the per-file cap, the per-source text budget or the budget shared with other sources. */
|
|
277
|
+
export type ReadBound = 'file' | 'source' | 'shared';
|
|
278
|
+
export interface ReadAllowance {
|
|
279
|
+
/** The most a file may hold: a read stops at one byte past it. */
|
|
280
|
+
readonly limit: number;
|
|
281
|
+
readonly bound: ReadBound;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* How much the next file may hold, given the bytes this source has already
|
|
285
|
+
* used: the smallest of the per-file cap and what is left of the per-source
|
|
286
|
+
* text budget and of the shared budget. The size seen at discovery is not part
|
|
287
|
+
* of it, because a file can grow after discovery.
|
|
288
|
+
*/
|
|
289
|
+
export declare function readAllowance(used: number, maxFileBytes: number, sharedBytes?: number): ReadAllowance;
|
|
290
|
+
/**
|
|
291
|
+
* Read the first `MAX_FILES` of `candidates` (already sorted), within the
|
|
292
|
+
* per-file cap and the per-call `MAX_TOTAL_BYTES`, and warn about whatever the
|
|
293
|
+
* file limit or the text budget left out. `truncated` is the discovery flag.
|
|
294
|
+
* Never throws.
|
|
295
|
+
*/
|
|
296
|
+
export declare function readSourceFiles<T>(options: ReadSourceFilesOptions<T>, candidates: readonly SourceFileCandidate[], truncated: boolean, warnings: string[]): SourceFilesRead<T>;
|
|
297
|
+
export type SourceFilesLoad<T> = {
|
|
298
|
+
readonly state: 'missing' | 'unreadable' | 'outside';
|
|
299
|
+
readonly warnings: string[];
|
|
300
|
+
} | {
|
|
301
|
+
readonly state: 'loaded';
|
|
302
|
+
readonly items: T[];
|
|
303
|
+
readonly bytesRead: number;
|
|
304
|
+
readonly warnings: string[];
|
|
305
|
+
};
|
|
306
|
+
/**
|
|
307
|
+
* The shared walk-and-read core: discover one directory, then read what the
|
|
308
|
+
* caps allow. `loadCorpus` is a caller of it for the built-in kinds; the
|
|
309
|
+
* external loader uses `discoverSourceFiles` and `readSourceFiles` directly so
|
|
310
|
+
* it can merge several roots and check a shared budget before reading.
|
|
311
|
+
*/
|
|
312
|
+
export declare function loadSourceFiles<T>(discover: DiscoverSourceFilesOptions, read: ReadSourceFilesOptions<T>): SourceFilesLoad<T>;
|
|
121
313
|
/**
|
|
122
314
|
* Load every indexable document of one kind under `root`, sorted by
|
|
123
315
|
* root-relative path in code-unit order. `root` should come from
|
|
124
316
|
* `resolveSearchRoot`.
|
|
125
317
|
*/
|
|
126
318
|
export declare function loadCorpus(root: string, kind?: DocumentKind): CorpusLoad;
|
|
319
|
+
export {};
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Deciduous reader behind `format: "deciduous"` external sources: turns a
|
|
3
|
+
* sync folder (`nodes/<change_id>.json` and `edges/<edge_id>.json`) into
|
|
4
|
+
* decision-shaped records with edge rationale as searchable text.
|
|
5
|
+
*
|
|
6
|
+
* It has its own scan instead of the shared `min(candidates, MAX_FILES)` cut:
|
|
7
|
+
* every root is probed first, then `nodes/` of every root in root order and
|
|
8
|
+
* then `edges/` of every root, in that explicit order and not a path sort.
|
|
9
|
+
* `MAX_DISCOVERED_FILES` bounds the files scanned, `MAX_LISTED_ENTRIES` bounds
|
|
10
|
+
* the entries one listing examines, `MAX_FILES` bounds the kept records, and
|
|
11
|
+
* the byte caps still apply, so edge files and filtered nodes never use up the
|
|
12
|
+
* record cap.
|
|
13
|
+
*
|
|
14
|
+
* Trust. File contents are untrusted. A file is opened only by a path built
|
|
15
|
+
* from a directory listing whose file stem passed the id charset, never from an
|
|
16
|
+
* id read out of content, and a display path is built only from that stem.
|
|
17
|
+
* Content ids (`change_id`, edge endpoints) are keys in maps and sets and
|
|
18
|
+
* nothing else. Symlinks are skipped at every depth. Reads files only.
|
|
19
|
+
*
|
|
20
|
+
* Import rule: node builtins, `search-bounded-read.ts` and `search-corpus.ts`
|
|
21
|
+
* only. It never imports the dispatcher (`search-external.ts`), which adapts
|
|
22
|
+
* its result.
|
|
23
|
+
*/
|
|
24
|
+
export type DeciduousState = 'loaded' | 'empty' | 'missing' | 'unreadable' | 'dropped';
|
|
25
|
+
/** A record as the dispatcher receives it; the dispatcher adds the source and origin. */
|
|
26
|
+
export interface DeciduousRecord {
|
|
27
|
+
/** `<source>/nodes/<id>.json`, built from a validated file stem. */
|
|
28
|
+
readonly path: string;
|
|
29
|
+
readonly title: string;
|
|
30
|
+
readonly nodeType: string;
|
|
31
|
+
readonly body: string;
|
|
32
|
+
}
|
|
33
|
+
export interface DeciduousLoad {
|
|
34
|
+
readonly state: DeciduousState;
|
|
35
|
+
readonly records: readonly DeciduousRecord[];
|
|
36
|
+
/** Bounded, sanitised, in deterministic order. They name the source, never a file or link target. */
|
|
37
|
+
readonly warnings: readonly string[];
|
|
38
|
+
/** Stat bytes of the files opened, the figure a shared read budget draws on. */
|
|
39
|
+
readonly bytesUsed: number;
|
|
40
|
+
}
|
|
41
|
+
export interface DeciduousLoadInput {
|
|
42
|
+
readonly name: string;
|
|
43
|
+
readonly roots: readonly string[];
|
|
44
|
+
/** The checkout root a project source must stay inside; unset for a user source. */
|
|
45
|
+
readonly containedIn?: string | undefined;
|
|
46
|
+
/** Bytes the source may open; a source that needs more is dropped whole, unread. */
|
|
47
|
+
readonly remainingBytes: number;
|
|
48
|
+
/** Largest single file read, in bytes. */
|
|
49
|
+
readonly maxFileBytes: number;
|
|
50
|
+
/** Directory entries the source may examine in total. Defaults to `MAX_LISTED_ENTRIES`. */
|
|
51
|
+
readonly entryLimit?: number | undefined;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Load one Deciduous source: scan `nodes/` then `edges/` of every root, read
|
|
55
|
+
* the nodes, keep the decision-shaped live ones, then fold edge rationale into
|
|
56
|
+
* the kept nodes. Never throws: a failure becomes a state and a bounded
|
|
57
|
+
* warning.
|
|
58
|
+
*/
|
|
59
|
+
export declare function loadDeciduousRecords(input: DeciduousLoadInput): DeciduousLoad;
|