@zenera/rag 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +144 -0
- package/dist/command.d.ts +3 -0
- package/dist/command.js +436 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +16 -0
- package/dist/present.d.ts +17 -0
- package/dist/present.js +40 -0
- package/dist/query.d.ts +7 -0
- package/dist/query.js +88 -0
- package/dist/repl.d.ts +8 -0
- package/dist/repl.js +119 -0
- package/dist/schema/build.d.ts +28 -0
- package/dist/schema/build.js +79 -0
- package/dist/schema/entities.d.ts +18 -0
- package/dist/schema/entities.js +71 -0
- package/dist/schema/files.d.ts +74 -0
- package/dist/schema/files.js +79 -0
- package/dist/schema/graph.d.ts +48 -0
- package/dist/schema/graph.js +320 -0
- package/dist/schema/hydrate.d.ts +19 -0
- package/dist/schema/hydrate.js +182 -0
- package/dist/schema/render.d.ts +11 -0
- package/dist/schema/render.js +254 -0
- package/dist/schema/schema.d.ts +11 -0
- package/dist/schema/schema.js +189 -0
- package/dist/schema/search.d.ts +50 -0
- package/dist/schema/search.js +142 -0
- package/dist/schema/spec.d.ts +58 -0
- package/dist/schema/spec.js +309 -0
- package/dist/schema/store.d.ts +32 -0
- package/dist/schema/store.js +126 -0
- package/dist/schema/subgraph.d.ts +42 -0
- package/dist/schema/subgraph.js +272 -0
- package/dist/schema/tools.d.ts +10 -0
- package/dist/schema/tools.js +242 -0
- package/dist/schema/typescript.d.ts +22 -0
- package/dist/schema/typescript.js +246 -0
- package/package.json +59 -0
package/dist/present.js
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { toOpenApi, toTypeScript } from "./schema/hydrate.js";
|
|
2
|
+
import { render } from "./schema/render.js";
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// One subgraph, in whichever of the four shapes was asked for
|
|
5
|
+
//
|
|
6
|
+
// The command, the prompt loop and the agent tool all print through here, so
|
|
7
|
+
// `--format ts` from a shell and `format: "ts"` from a model are the same
|
|
8
|
+
// bytes. Two of the four need the schemas off disk and two do not, which is
|
|
9
|
+
// the only reason this is async.
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
export const FORMATS = ['text', 'mermaid', 'mermaid-flowchart', 'ts', 'openapi'];
|
|
12
|
+
export function isFormat(value) {
|
|
13
|
+
return FORMATS.includes(value);
|
|
14
|
+
}
|
|
15
|
+
export async function present(index, subgraphs, format, options = {}) {
|
|
16
|
+
if (subgraphs.length === 0) {
|
|
17
|
+
return format === 'openapi' ? '{}' : '';
|
|
18
|
+
}
|
|
19
|
+
if (format === 'ts') {
|
|
20
|
+
const schemas = await index.schemas();
|
|
21
|
+
return subgraphs.map((s) => toTypeScript(s, schemas, options)).join('\n');
|
|
22
|
+
}
|
|
23
|
+
if (format === 'openapi') {
|
|
24
|
+
const [schemas, operations] = await Promise.all([index.schemas(), index.operations()]);
|
|
25
|
+
// One document for the whole answer: several would each have to repeat
|
|
26
|
+
// the components the others also touched.
|
|
27
|
+
const merged = subgraphs.reduce((all, one) => ({
|
|
28
|
+
nodes: [...all.nodes, ...one.nodes],
|
|
29
|
+
edges: [...all.edges, ...one.edges],
|
|
30
|
+
hits: [...all.hits, ...one.hits],
|
|
31
|
+
score: all.score + one.score,
|
|
32
|
+
truncated: all.truncated || one.truncated,
|
|
33
|
+
}), { nodes: [], edges: [], hits: [], score: 0, truncated: false });
|
|
34
|
+
return JSON.stringify(toOpenApi(merged, schemas, operations), null, 2);
|
|
35
|
+
}
|
|
36
|
+
return subgraphs
|
|
37
|
+
.map((s, at) => `${subgraphs.length > 1 ? `# result ${at + 1}\n` : ''}${render(s, format, options)}`)
|
|
38
|
+
.join('\n\n');
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=present.js.map
|
package/dist/query.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SchemaQuery } from './schema/search.ts';
|
|
2
|
+
export declare class QueryError extends Error {
|
|
3
|
+
}
|
|
4
|
+
export declare function parseQuery(value: unknown): SchemaQuery;
|
|
5
|
+
/** Whether anything was actually asked, as opposed to only filtered. */
|
|
6
|
+
export declare function isEmpty(query: SchemaQuery): boolean;
|
|
7
|
+
//# sourceMappingURL=query.d.ts.map
|
package/dist/query.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// A query, from somewhere untrusted
|
|
3
|
+
//
|
|
4
|
+
// `--query` and the tool both hand over a whole object, which means the shape
|
|
5
|
+
// has to be checked rather than assumed. The rule is the same either way: a
|
|
6
|
+
// key that is not known is an error, because a silently ignored
|
|
7
|
+
// `output_propertys` looks exactly like a search that found nothing.
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
const LISTS = [
|
|
10
|
+
'all',
|
|
11
|
+
'methods',
|
|
12
|
+
'types',
|
|
13
|
+
'input_types',
|
|
14
|
+
'output_types',
|
|
15
|
+
'properties',
|
|
16
|
+
'input_properties',
|
|
17
|
+
'output_properties',
|
|
18
|
+
'exclude_ids',
|
|
19
|
+
'exclude_methods',
|
|
20
|
+
'exclude_types',
|
|
21
|
+
'exclude_properties',
|
|
22
|
+
];
|
|
23
|
+
const NUMBERS = ['limit', 'max_hops', 'max_nodes'];
|
|
24
|
+
const DIRECTIONS = ['input', 'output', 'any'];
|
|
25
|
+
const METHOD_TYPES = ['read_only', 'read_write', 'any'];
|
|
26
|
+
const KNOWN = new Set([...LISTS, ...NUMBERS, 'direction', 'method_type']);
|
|
27
|
+
export class QueryError extends Error {
|
|
28
|
+
}
|
|
29
|
+
export function parseQuery(value) {
|
|
30
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
31
|
+
throw new QueryError('a query must be an object');
|
|
32
|
+
}
|
|
33
|
+
const input = value;
|
|
34
|
+
for (const key of Object.keys(input)) {
|
|
35
|
+
if (!KNOWN.has(key)) {
|
|
36
|
+
throw new QueryError(`unknown query field "${key}" — expected ${[...KNOWN].join(', ')}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const out = {};
|
|
40
|
+
for (const key of LISTS) {
|
|
41
|
+
const list = input[key];
|
|
42
|
+
if (list === undefined) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (!Array.isArray(list) || list.some((v) => typeof v !== 'string')) {
|
|
46
|
+
throw new QueryError(`${key} must be an array of strings`);
|
|
47
|
+
}
|
|
48
|
+
out[key] = list;
|
|
49
|
+
}
|
|
50
|
+
for (const key of NUMBERS) {
|
|
51
|
+
const number = input[key];
|
|
52
|
+
if (number === undefined) {
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (typeof number !== 'number' || !Number.isInteger(number) || number < 1) {
|
|
56
|
+
throw new QueryError(`${key} must be a whole number of at least 1`);
|
|
57
|
+
}
|
|
58
|
+
out[key] = number;
|
|
59
|
+
}
|
|
60
|
+
if (input.direction !== undefined) {
|
|
61
|
+
out.direction = oneOf('direction', input.direction, DIRECTIONS);
|
|
62
|
+
}
|
|
63
|
+
if (input.method_type !== undefined) {
|
|
64
|
+
out.method_type = oneOf('method_type', input.method_type, METHOD_TYPES);
|
|
65
|
+
}
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
/** Whether anything was actually asked, as opposed to only filtered. */
|
|
69
|
+
export function isEmpty(query) {
|
|
70
|
+
const terms = [
|
|
71
|
+
query.all,
|
|
72
|
+
query.methods,
|
|
73
|
+
query.types,
|
|
74
|
+
query.input_types,
|
|
75
|
+
query.output_types,
|
|
76
|
+
query.properties,
|
|
77
|
+
query.input_properties,
|
|
78
|
+
query.output_properties,
|
|
79
|
+
];
|
|
80
|
+
return terms.every((list) => !list || list.length === 0);
|
|
81
|
+
}
|
|
82
|
+
function oneOf(key, value, allowed) {
|
|
83
|
+
if (typeof value !== 'string' || !allowed.includes(value)) {
|
|
84
|
+
throw new QueryError(`${key} must be one of ${allowed.join(', ')}`);
|
|
85
|
+
}
|
|
86
|
+
return value;
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=query.js.map
|
package/dist/repl.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type Format, type OutputOptions } from './present.ts';
|
|
2
|
+
import type { SchemaIndex, SchemaQuery } from './schema/search.ts';
|
|
3
|
+
interface Settings extends OutputOptions {
|
|
4
|
+
format: Format;
|
|
5
|
+
}
|
|
6
|
+
export declare function repl(index: SchemaIndex, initial: SchemaQuery, settings: Settings): Promise<void>;
|
|
7
|
+
export {};
|
|
8
|
+
//# sourceMappingURL=repl.d.ts.map
|
package/dist/repl.js
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { createInterface } from 'node:readline/promises';
|
|
2
|
+
import { bold, cyan, dim, note, write } from '@zenera/cli/lib';
|
|
3
|
+
import { isFormat, present } from "./present.js";
|
|
4
|
+
import { isEmpty, parseQuery, QueryError } from "./query.js";
|
|
5
|
+
const FIELDS = {
|
|
6
|
+
all: 'all',
|
|
7
|
+
method: 'methods',
|
|
8
|
+
type: 'types',
|
|
9
|
+
'input-type': 'input_types',
|
|
10
|
+
'output-type': 'output_types',
|
|
11
|
+
property: 'properties',
|
|
12
|
+
'input-property': 'input_properties',
|
|
13
|
+
'output-property': 'output_properties',
|
|
14
|
+
};
|
|
15
|
+
export async function repl(index, initial, settings) {
|
|
16
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
17
|
+
let query = { ...initial };
|
|
18
|
+
let format = settings.format;
|
|
19
|
+
const options = { docs: settings.docs, onlyHits: settings.onlyHits };
|
|
20
|
+
note(bold(`${index.manifest.sources.map((s) => s.title).join(', ')}`));
|
|
21
|
+
note(dim(' a bare line searches every kind; `help` lists the rest.'));
|
|
22
|
+
try {
|
|
23
|
+
if (!isEmpty(query)) {
|
|
24
|
+
query = await run(index, query, format, options);
|
|
25
|
+
}
|
|
26
|
+
for (;;) {
|
|
27
|
+
const line = (await rl.question(cyan('rag> '))).trim();
|
|
28
|
+
if (line === '') {
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
if (line === 'quit' || line === 'exit') {
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (line === 'help') {
|
|
35
|
+
help();
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
if (line === 'reset') {
|
|
39
|
+
query = {};
|
|
40
|
+
note(dim(' cleared, exclusions and all'));
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if (line === 'show') {
|
|
44
|
+
note(dim(` ${JSON.stringify(query)}`));
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const [head, ...rest] = line.split(' ');
|
|
48
|
+
const tail = rest.join(' ').trim();
|
|
49
|
+
if (head === 'format') {
|
|
50
|
+
if (isFormat(tail)) {
|
|
51
|
+
format = tail;
|
|
52
|
+
note(dim(` format is ${tail}`));
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
note(dim(' expected text, mermaid, mermaid-flowchart, ts or openapi'));
|
|
56
|
+
}
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (head === 'direction' || head === 'method-type') {
|
|
60
|
+
query = patch(query, head === 'direction' ? 'direction' : 'method_type', tail);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (head && head in FIELDS && tail) {
|
|
64
|
+
query = { ...query, [FIELDS[head]]: [tail] };
|
|
65
|
+
query = await run(index, query, format, options);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
query = await run(index, { ...query, all: [line] }, format, options);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
rl.close();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Runs the search, prints it, and hands back the query with everything just
|
|
77
|
+
* shown added to the exclusions — so asking the same thing twice moves on
|
|
78
|
+
* instead of repeating itself.
|
|
79
|
+
*/
|
|
80
|
+
async function run(index, query, format, options) {
|
|
81
|
+
const result = await index.search(query);
|
|
82
|
+
const text = await present(index, result.subgraphs, format, options);
|
|
83
|
+
if (text) {
|
|
84
|
+
write(text);
|
|
85
|
+
}
|
|
86
|
+
note(dim(` ${result.seeds.length} seed(s) · ${result.subgraphs.length} result(s)` +
|
|
87
|
+
(result.empty.length > 0 ? ` · nothing for: ${result.empty.join(', ')}` : '')));
|
|
88
|
+
if (result.seeds.length === 0) {
|
|
89
|
+
return query;
|
|
90
|
+
}
|
|
91
|
+
const seen = new Set([...(query.exclude_ids ?? []), ...result.seeds.map((s) => s.id)]);
|
|
92
|
+
note(dim(` ${seen.size} node(s) now excluded — \`reset\` to see them again`));
|
|
93
|
+
return { ...query, exclude_ids: [...seen] };
|
|
94
|
+
}
|
|
95
|
+
function patch(query, key, value) {
|
|
96
|
+
try {
|
|
97
|
+
return { ...query, ...parseQuery({ [key]: value }) };
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
note(dim(` ${err instanceof QueryError ? err.message : String(err)}`));
|
|
101
|
+
return query;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
function help() {
|
|
105
|
+
for (const line of [
|
|
106
|
+
' <text> search everything',
|
|
107
|
+
' all|method|type <text> search one field',
|
|
108
|
+
' input-property <text> also: output-property, property, input-type, output-type',
|
|
109
|
+
' direction <d> input | output | any',
|
|
110
|
+
' method-type <t> read_only | read_write | any',
|
|
111
|
+
' format <f> text | mermaid | mermaid-flowchart | ts | openapi',
|
|
112
|
+
' show the query as it stands',
|
|
113
|
+
' reset forget it, exclusions included',
|
|
114
|
+
' quit',
|
|
115
|
+
]) {
|
|
116
|
+
note(dim(line));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=repl.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Embedder } from '@zenera/neo';
|
|
2
|
+
import { type EntityRecord } from './entities.ts';
|
|
3
|
+
import { type Counts, type Manifest, type SourceRecord } from './files.ts';
|
|
4
|
+
export interface BuildOptions {
|
|
5
|
+
files: readonly string[];
|
|
6
|
+
out: string;
|
|
7
|
+
embedder: Embedder;
|
|
8
|
+
/** the reference as it was written, which is what a later search will type */
|
|
9
|
+
embeddingRef?: string;
|
|
10
|
+
/** told the manifest, so a store can say what wrote it */
|
|
11
|
+
indexer: string;
|
|
12
|
+
/** texts sent to the embedder at once */
|
|
13
|
+
batch?: number;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
/** what the documents turned out to hold, before a vector has been paid for */
|
|
16
|
+
onRead?: (summary: BuildSummary) => void;
|
|
17
|
+
onProgress?: (done: number, total: number) => void;
|
|
18
|
+
}
|
|
19
|
+
export interface BuildSummary {
|
|
20
|
+
sources: SourceRecord[];
|
|
21
|
+
counts: Counts;
|
|
22
|
+
}
|
|
23
|
+
export interface BuildResult {
|
|
24
|
+
manifest: Manifest;
|
|
25
|
+
entities: EntityRecord[];
|
|
26
|
+
}
|
|
27
|
+
export declare function buildIndex(options: BuildOptions): Promise<BuildResult>;
|
|
28
|
+
//# sourceMappingURL=build.d.ts.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { toEntities } from "./entities.js";
|
|
2
|
+
import { INDEX_VERSION, writeIndex, } from "./files.js";
|
|
3
|
+
import { buildGraph } from "./graph.js";
|
|
4
|
+
import { loadSpecs } from "./spec.js";
|
|
5
|
+
import { writeStore } from "./store.js";
|
|
6
|
+
const DEFAULT_BATCH = 96;
|
|
7
|
+
export async function buildIndex(options) {
|
|
8
|
+
const corpus = await loadSpecs(options.files);
|
|
9
|
+
const { graph, types } = buildGraph(corpus);
|
|
10
|
+
const entities = toEntities(graph);
|
|
11
|
+
const summary = {
|
|
12
|
+
sources: sourcesOf(corpus, entities, options.files),
|
|
13
|
+
counts: {
|
|
14
|
+
methods: entities.filter((e) => e.kind === 'method').length,
|
|
15
|
+
types: entities.filter((e) => e.kind === 'type').length,
|
|
16
|
+
properties: entities.filter((e) => e.kind === 'property').length,
|
|
17
|
+
entities: entities.length,
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
options.onRead?.(summary);
|
|
21
|
+
const vectors = await embedAll(entities, options);
|
|
22
|
+
const written = await writeStore(options.out, entities, vectors);
|
|
23
|
+
const manifest = {
|
|
24
|
+
version: INDEX_VERSION,
|
|
25
|
+
createdAt: new Date().toISOString(),
|
|
26
|
+
indexer: options.indexer,
|
|
27
|
+
embedding: {
|
|
28
|
+
ref: options.embeddingRef ?? options.embedder.id,
|
|
29
|
+
id: options.embedder.id,
|
|
30
|
+
dimensions: vectors[0]?.length ?? 0,
|
|
31
|
+
},
|
|
32
|
+
sources: summary.sources,
|
|
33
|
+
counts: summary.counts,
|
|
34
|
+
indexes: { fts: written.fts, vector: written.vector },
|
|
35
|
+
};
|
|
36
|
+
await writeIndex(options.out, { manifest, graph, types, operations: corpus.operations });
|
|
37
|
+
return { manifest, entities };
|
|
38
|
+
}
|
|
39
|
+
async function embedAll(entities, options) {
|
|
40
|
+
const size = options.batch ?? DEFAULT_BATCH;
|
|
41
|
+
const out = [];
|
|
42
|
+
for (let at = 0; at < entities.length; at += size) {
|
|
43
|
+
const slice = entities.slice(at, at + size);
|
|
44
|
+
const response = await options.embedder.embed({
|
|
45
|
+
input: slice.map((e) => e.text),
|
|
46
|
+
taskType: 'document',
|
|
47
|
+
signal: options.signal,
|
|
48
|
+
});
|
|
49
|
+
if (response.vectors.length !== slice.length) {
|
|
50
|
+
throw new Error(`${options.embedder.id} answered ${response.vectors.length} vectors for ${slice.length} texts`);
|
|
51
|
+
}
|
|
52
|
+
out.push(...response.vectors.map((v) => Float32Array.from(v)));
|
|
53
|
+
options.onProgress?.(out.length, entities.length);
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Counted off the entities rather than the corpus, so a schema the document
|
|
59
|
+
* never named — an inline request body, say — is credited to the file it came
|
|
60
|
+
* from instead of going missing.
|
|
61
|
+
*/
|
|
62
|
+
function sourcesOf(corpus, entities, files) {
|
|
63
|
+
return corpus.docs.map((doc, index) => {
|
|
64
|
+
const mine = entities.filter((e) => e.source === doc.source);
|
|
65
|
+
const operations = corpus.operations.filter((op) => op.source === doc.source);
|
|
66
|
+
return {
|
|
67
|
+
path: files[index] ?? doc.source,
|
|
68
|
+
sha256: doc.sha256,
|
|
69
|
+
dialect: doc.dialect,
|
|
70
|
+
title: doc.title,
|
|
71
|
+
version: doc.version,
|
|
72
|
+
paths: new Set(operations.map((op) => op.path)).size,
|
|
73
|
+
methods: operations.length,
|
|
74
|
+
types: mine.filter((e) => e.kind === 'type').length,
|
|
75
|
+
properties: mine.filter((e) => e.kind === 'property').length,
|
|
76
|
+
};
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=build.js.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ApiGraph } from './graph.ts';
|
|
2
|
+
export interface EntityRecord {
|
|
3
|
+
id: string;
|
|
4
|
+
kind: string;
|
|
5
|
+
name: string;
|
|
6
|
+
parent: string;
|
|
7
|
+
doc: string;
|
|
8
|
+
direction: string;
|
|
9
|
+
methodType: string;
|
|
10
|
+
httpMethod: string;
|
|
11
|
+
path: string;
|
|
12
|
+
source: string;
|
|
13
|
+
signature: string;
|
|
14
|
+
required: boolean;
|
|
15
|
+
text: string;
|
|
16
|
+
}
|
|
17
|
+
export declare function toEntities(graph: ApiGraph): EntityRecord[];
|
|
18
|
+
//# sourceMappingURL=entities.d.ts.map
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/** How many field names of a type are worth putting in its search text. */
|
|
2
|
+
const FIELDS_IN_TEXT = 40;
|
|
3
|
+
export function toEntities(graph) {
|
|
4
|
+
const out = [];
|
|
5
|
+
for (const id of graph.nodes()) {
|
|
6
|
+
const a = graph.getNodeAttributes(id);
|
|
7
|
+
out.push({
|
|
8
|
+
id,
|
|
9
|
+
kind: a.kind,
|
|
10
|
+
name: a.name,
|
|
11
|
+
parent: a.parent,
|
|
12
|
+
doc: a.doc,
|
|
13
|
+
direction: a.direction,
|
|
14
|
+
methodType: a.methodType,
|
|
15
|
+
httpMethod: a.httpMethod,
|
|
16
|
+
path: a.path,
|
|
17
|
+
source: a.source,
|
|
18
|
+
signature: a.signature,
|
|
19
|
+
required: a.required,
|
|
20
|
+
text: textOf(graph, id),
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
return out.sort((a, b) => a.id.localeCompare(b.id));
|
|
24
|
+
}
|
|
25
|
+
function textOf(graph, id) {
|
|
26
|
+
const a = graph.getNodeAttributes(id);
|
|
27
|
+
const parts = [];
|
|
28
|
+
if (a.kind === 'method') {
|
|
29
|
+
parts.push(`[method] ${a.httpMethod} ${a.path}`, a.name, a.methodType.replace('_', ' '));
|
|
30
|
+
}
|
|
31
|
+
else if (a.kind === 'type') {
|
|
32
|
+
parts.push(`[type] ${a.name}`, side(a.direction));
|
|
33
|
+
const fields = graph
|
|
34
|
+
.outEdges(id)
|
|
35
|
+
.filter((e) => graph.getEdgeAttribute(e, 'relation') === 'HAS_PROPERTY')
|
|
36
|
+
.slice(0, FIELDS_IN_TEXT)
|
|
37
|
+
.map((e) => graph.getNodeAttribute(graph.target(e), 'name'));
|
|
38
|
+
if (fields.length > 0) {
|
|
39
|
+
parts.push(`fields: ${fields.join(', ')}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
const where = a.parent ? ` in ${a.parent}` : '';
|
|
44
|
+
parts.push(`[property] ${a.name}${a.signature ? `: ${a.signature}` : ''}${where}`);
|
|
45
|
+
parts.push(side(a.direction));
|
|
46
|
+
}
|
|
47
|
+
const spaced = words(a.name);
|
|
48
|
+
if (spaced) {
|
|
49
|
+
parts.push(spaced);
|
|
50
|
+
}
|
|
51
|
+
if (a.doc) {
|
|
52
|
+
parts.push(`— ${a.doc}`);
|
|
53
|
+
}
|
|
54
|
+
return parts.filter(Boolean).join(' ').replace(/\s+/g, ' ').trim();
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* `meowVolume` as "meow volume". No tokenizer splits an identifier, so without
|
|
58
|
+
* this the only way to a field is to already know how it was capitalized.
|
|
59
|
+
*/
|
|
60
|
+
function words(name) {
|
|
61
|
+
const spaced = name
|
|
62
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
63
|
+
.replace(/[_\-.]+/g, ' ')
|
|
64
|
+
.toLowerCase()
|
|
65
|
+
.trim();
|
|
66
|
+
return spaced === name.toLowerCase() ? '' : spaced;
|
|
67
|
+
}
|
|
68
|
+
function side(direction) {
|
|
69
|
+
return direction === 'none' ? '' : `(${direction})`;
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=entities.js.map
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { ApiGraph } from './graph.ts';
|
|
2
|
+
import type { Schema } from './schema.ts';
|
|
3
|
+
import type { Operation } from './spec.ts';
|
|
4
|
+
export declare const INDEX_VERSION = 1;
|
|
5
|
+
export declare const MANIFEST_FILE = "manifest.json";
|
|
6
|
+
export declare const GRAPH_FILE = "graph.json";
|
|
7
|
+
export declare const SCHEMAS_FILE = "schemas.json";
|
|
8
|
+
export declare const OPERATIONS_FILE = "operations.json";
|
|
9
|
+
export declare const LANCE_DIR = "lance";
|
|
10
|
+
export interface SourceRecord {
|
|
11
|
+
path: string;
|
|
12
|
+
sha256: string;
|
|
13
|
+
dialect: string;
|
|
14
|
+
title: string;
|
|
15
|
+
version: string;
|
|
16
|
+
paths: number;
|
|
17
|
+
methods: number;
|
|
18
|
+
types: number;
|
|
19
|
+
properties: number;
|
|
20
|
+
}
|
|
21
|
+
export interface Counts {
|
|
22
|
+
methods: number;
|
|
23
|
+
types: number;
|
|
24
|
+
properties: number;
|
|
25
|
+
entities: number;
|
|
26
|
+
}
|
|
27
|
+
export interface Manifest {
|
|
28
|
+
version: number;
|
|
29
|
+
createdAt: string;
|
|
30
|
+
indexer: string;
|
|
31
|
+
/** `ref` as it was typed, `id` as the embedder answers to it */
|
|
32
|
+
embedding: {
|
|
33
|
+
ref: string;
|
|
34
|
+
id: string;
|
|
35
|
+
dimensions: number;
|
|
36
|
+
};
|
|
37
|
+
sources: SourceRecord[];
|
|
38
|
+
counts: Counts;
|
|
39
|
+
/** whether the table carries an fts index, and whether it carries a vector one */
|
|
40
|
+
indexes: {
|
|
41
|
+
fts: boolean;
|
|
42
|
+
vector: boolean;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export interface WrittenIndex {
|
|
46
|
+
manifest: Manifest;
|
|
47
|
+
graph: ApiGraph;
|
|
48
|
+
types: Readonly<Record<string, Schema>>;
|
|
49
|
+
operations: readonly Operation[];
|
|
50
|
+
}
|
|
51
|
+
export interface OpenIndex {
|
|
52
|
+
dir: string;
|
|
53
|
+
manifest: Manifest;
|
|
54
|
+
graph: ApiGraph;
|
|
55
|
+
/** the raw schemas, read once and remembered */
|
|
56
|
+
schemas(): Promise<Record<string, Schema>>;
|
|
57
|
+
/** the operations, likewise */
|
|
58
|
+
operations(): Promise<Operation[]>;
|
|
59
|
+
}
|
|
60
|
+
export declare const lancePath: (dir: string) => string;
|
|
61
|
+
export declare function writeIndex(dir: string, index: WrittenIndex): Promise<void>;
|
|
62
|
+
export declare function openIndex(dir: string): Promise<OpenIndex>;
|
|
63
|
+
export declare function readManifest(dir: string): Promise<Manifest>;
|
|
64
|
+
/**
|
|
65
|
+
* A store answers with the neighbours of a vector, and a vector means nothing
|
|
66
|
+
* without the model that produced it. Asking one model's index a question
|
|
67
|
+
* embedded by another returns rows, in an order that is noise.
|
|
68
|
+
*
|
|
69
|
+
* Either spelling is accepted, because `openai:text-embedding-3-small` and
|
|
70
|
+
* `text-embedding-3-small` are one model and which of them was typed is not
|
|
71
|
+
* something anyone should have to remember.
|
|
72
|
+
*/
|
|
73
|
+
export declare function assertSameEmbedding(manifest: Manifest, ref: string): void;
|
|
74
|
+
//# sourceMappingURL=files.d.ts.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { MultiDirectedGraph } from 'graphology';
|
|
2
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { CliError, EXIT } from '@zenera/cli/lib';
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// What an index is, on disk
|
|
7
|
+
//
|
|
8
|
+
// Four things in a directory, and the order they are written in is the whole
|
|
9
|
+
// crash story: the manifest goes last, so a half-built index has no manifest
|
|
10
|
+
// and reads as "not indexed" rather than as a store that quietly lost half its
|
|
11
|
+
// operations.
|
|
12
|
+
//
|
|
13
|
+
// `graph.json` is deliberately thin — names, directions, edges — because it is
|
|
14
|
+
// parsed whole on every search. The schemas are the bulk of the bytes and are
|
|
15
|
+
// wanted only when something is being printed, so they live apart and are read
|
|
16
|
+
// on first use.
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
export const INDEX_VERSION = 1;
|
|
19
|
+
export const MANIFEST_FILE = 'manifest.json';
|
|
20
|
+
export const GRAPH_FILE = 'graph.json';
|
|
21
|
+
export const SCHEMAS_FILE = 'schemas.json';
|
|
22
|
+
export const OPERATIONS_FILE = 'operations.json';
|
|
23
|
+
export const LANCE_DIR = 'lance';
|
|
24
|
+
export const lancePath = (dir) => join(dir, LANCE_DIR);
|
|
25
|
+
export async function writeIndex(dir, index) {
|
|
26
|
+
await mkdir(dir, { recursive: true });
|
|
27
|
+
await writeFile(join(dir, GRAPH_FILE), JSON.stringify(index.graph.export()));
|
|
28
|
+
await writeFile(join(dir, SCHEMAS_FILE), JSON.stringify(index.types));
|
|
29
|
+
await writeFile(join(dir, OPERATIONS_FILE), JSON.stringify(index.operations));
|
|
30
|
+
await writeFile(join(dir, MANIFEST_FILE), `${JSON.stringify(index.manifest, null, 4)}\n`);
|
|
31
|
+
}
|
|
32
|
+
export async function openIndex(dir) {
|
|
33
|
+
const manifest = await readManifest(dir);
|
|
34
|
+
const graph = new MultiDirectedGraph();
|
|
35
|
+
graph.import(JSON.parse(await readFile(join(dir, GRAPH_FILE), 'utf8')));
|
|
36
|
+
return {
|
|
37
|
+
dir,
|
|
38
|
+
manifest,
|
|
39
|
+
graph,
|
|
40
|
+
schemas: once(() => read(dir, SCHEMAS_FILE)),
|
|
41
|
+
operations: once(() => read(dir, OPERATIONS_FILE)),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
async function read(dir, file) {
|
|
45
|
+
return JSON.parse(await readFile(join(dir, file), 'utf8'));
|
|
46
|
+
}
|
|
47
|
+
function once(load) {
|
|
48
|
+
let pending;
|
|
49
|
+
return () => (pending ??= load());
|
|
50
|
+
}
|
|
51
|
+
export async function readManifest(dir) {
|
|
52
|
+
let text;
|
|
53
|
+
try {
|
|
54
|
+
text = await readFile(join(dir, MANIFEST_FILE), 'utf8');
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
throw new CliError(`${dir} does not hold an index`, EXIT.invalid, 'build one first with `zen rag schema index`');
|
|
58
|
+
}
|
|
59
|
+
const manifest = JSON.parse(text);
|
|
60
|
+
if (manifest.version !== INDEX_VERSION) {
|
|
61
|
+
throw new CliError(`${dir} was written by another version of this indexer`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
|
|
62
|
+
}
|
|
63
|
+
return manifest;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* A store answers with the neighbours of a vector, and a vector means nothing
|
|
67
|
+
* without the model that produced it. Asking one model's index a question
|
|
68
|
+
* embedded by another returns rows, in an order that is noise.
|
|
69
|
+
*
|
|
70
|
+
* Either spelling is accepted, because `openai:text-embedding-3-small` and
|
|
71
|
+
* `text-embedding-3-small` are one model and which of them was typed is not
|
|
72
|
+
* something anyone should have to remember.
|
|
73
|
+
*/
|
|
74
|
+
export function assertSameEmbedding(manifest, ref) {
|
|
75
|
+
if (ref !== manifest.embedding.ref && ref !== manifest.embedding.id) {
|
|
76
|
+
throw new CliError(`this index was built with ${manifest.embedding.ref}, not ${ref}`, EXIT.invalid, `search it with --embedding ${manifest.embedding.ref}, or rebuild it`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=files.js.map
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { MultiDirectedGraph } from 'graphology';
|
|
2
|
+
import { type Schema } from './schema.ts';
|
|
3
|
+
import { type Corpus, type MethodType, type ParamIn } from './spec.ts';
|
|
4
|
+
export type NodeKind = 'method' | 'type' | 'property';
|
|
5
|
+
export type Direction = 'input' | 'output' | 'both' | 'none';
|
|
6
|
+
export type Relation = 'TAKES_INPUT' | 'HAS_PARAM' | 'RETURNS_OUTPUT' | 'HAS_PROPERTY' | 'OF_TYPE' | 'COMPOSES' | 'ITEM_OF';
|
|
7
|
+
export interface NodeAttrs {
|
|
8
|
+
kind: NodeKind;
|
|
9
|
+
name: string;
|
|
10
|
+
/** owning type id for a property, operation id for a parameter, else '' */
|
|
11
|
+
parent: string;
|
|
12
|
+
doc: string;
|
|
13
|
+
direction: Direction;
|
|
14
|
+
methodType: MethodType | 'n/a';
|
|
15
|
+
httpMethod: string;
|
|
16
|
+
path: string;
|
|
17
|
+
source: string;
|
|
18
|
+
/** the TypeScript type expression, for properties */
|
|
19
|
+
signature: string;
|
|
20
|
+
required: boolean;
|
|
21
|
+
}
|
|
22
|
+
export interface EdgeAttrs {
|
|
23
|
+
relation: Relation;
|
|
24
|
+
/** on RETURNS_OUTPUT */
|
|
25
|
+
status: number;
|
|
26
|
+
/** on HAS_PARAM */
|
|
27
|
+
in: ParamIn | '';
|
|
28
|
+
}
|
|
29
|
+
export type ApiGraph = MultiDirectedGraph<NodeAttrs, EdgeAttrs>;
|
|
30
|
+
export interface Built {
|
|
31
|
+
graph: ApiGraph;
|
|
32
|
+
/** every type the graph names, synthesized ones included */
|
|
33
|
+
types: Record<string, Schema>;
|
|
34
|
+
}
|
|
35
|
+
export declare const methodId: (operationId: string) => string;
|
|
36
|
+
export declare const typeId: (name: string) => string;
|
|
37
|
+
export declare const propertyId: (parent: string, name: string) => string;
|
|
38
|
+
export declare const paramId: (operationId: string, name: string) => string;
|
|
39
|
+
export declare function buildGraph(corpus: Corpus): Built;
|
|
40
|
+
/**
|
|
41
|
+
* Which side of a call a node lives on. Seeded from the operations — a request
|
|
42
|
+
* body is an input, a response is an output — and then pushed down through
|
|
43
|
+
* composition until it stops. A DTO used both ways comes out `both`, which is
|
|
44
|
+
* the honest answer and the reason this is a closure rather than a flag set at
|
|
45
|
+
* the point of use.
|
|
46
|
+
*/
|
|
47
|
+
export declare function propagate(graph: ApiGraph): void;
|
|
48
|
+
//# sourceMappingURL=graph.d.ts.map
|