@remnic/coding-graph 9.3.759
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -0
- package/dist/chunk-5I2DBHOQ.js +1042 -0
- package/dist/chunk-5I2DBHOQ.js.map +1 -0
- package/dist/chunk-CPYJACC5.js +1838 -0
- package/dist/chunk-CPYJACC5.js.map +1 -0
- package/dist/chunk-ZVCMIM4T.js +216 -0
- package/dist/chunk-ZVCMIM4T.js.map +1 -0
- package/dist/cypher/query-parser.d.ts +253 -0
- package/dist/cypher/query-parser.js +17 -0
- package/dist/cypher/query-parser.js.map +1 -0
- package/dist/graph-schema.d.ts +84 -0
- package/dist/graph-schema.js +17 -0
- package/dist/graph-schema.js.map +1 -0
- package/dist/graph-store.d.ts +938 -0
- package/dist/graph-store.js +16 -0
- package/dist/graph-store.js.map +1 -0
- package/dist/index.d.ts +1953 -0
- package/dist/index.js +3509 -0
- package/dist/index.js.map +1 -0
- package/grammars/tree-sitter-bash.wasm +0 -0
- package/grammars/tree-sitter-c.wasm +0 -0
- package/grammars/tree-sitter-c_sharp.wasm +0 -0
- package/grammars/tree-sitter-cpp.wasm +0 -0
- package/grammars/tree-sitter-go.wasm +0 -0
- package/grammars/tree-sitter-java.wasm +0 -0
- package/grammars/tree-sitter-javascript.wasm +0 -0
- package/grammars/tree-sitter-kotlin.wasm +0 -0
- package/grammars/tree-sitter-php.wasm +0 -0
- package/grammars/tree-sitter-python.wasm +0 -0
- package/grammars/tree-sitter-ruby.wasm +0 -0
- package/grammars/tree-sitter-rust.wasm +0 -0
- package/grammars/tree-sitter-swift.wasm +0 -0
- package/grammars/tree-sitter-tsx.wasm +0 -0
- package/grammars/tree-sitter-typescript.wasm +0 -0
- package/package.json +79 -0
- package/src/co-change.test.ts +175 -0
- package/src/co-change.ts +167 -0
- package/src/cypher/query-parser.test.ts +1107 -0
- package/src/cypher/query-parser.ts +1692 -0
- package/src/detect-changes.test.ts +533 -0
- package/src/detect-changes.ts +367 -0
- package/src/engine/emit.ts +556 -0
- package/src/engine/engine.test.ts +1417 -0
- package/src/engine/engine.ts +182 -0
- package/src/engine/extractors.ts +486 -0
- package/src/engine/fixtures.ts +364 -0
- package/src/engine/language-sniff.ts +56 -0
- package/src/engine/parser-backend.ts +206 -0
- package/src/engine/utf16-offsets.ts +68 -0
- package/src/git-invoker.test.ts +116 -0
- package/src/git-invoker.ts +426 -0
- package/src/graph-schema.test.ts +541 -0
- package/src/graph-schema.ts +383 -0
- package/src/graph-store-pr2.test.ts +1879 -0
- package/src/graph-store.test.ts +1420 -0
- package/src/graph-store.ts +3489 -0
- package/src/index-status.test.ts +303 -0
- package/src/index-status.ts +135 -0
- package/src/index.ts +384 -0
- package/src/lsp/byte-position.ts +173 -0
- package/src/lsp/characterization.test.ts +174 -0
- package/src/lsp/client.test.ts +275 -0
- package/src/lsp/client.ts +484 -0
- package/src/lsp/config.ts +219 -0
- package/src/lsp/degradation.ts +86 -0
- package/src/lsp/fixtures/fake-server.mjs +198 -0
- package/src/lsp/framing.test.ts +180 -0
- package/src/lsp/framing.ts +177 -0
- package/src/lsp/resolution.test.ts +497 -0
- package/src/lsp/resolution.ts +483 -0
- package/src/lsp/status.ts +140 -0
- package/src/lsp/types.ts +167 -0
- package/src/reindex.test.ts +1038 -0
- package/src/reindex.ts +908 -0
- package/src/row-types.ts +45 -0
- package/src/semantic/canonical-text.test.ts +150 -0
- package/src/semantic/canonical-text.ts +219 -0
- package/src/semantic/config.ts +235 -0
- package/src/semantic/index.ts +78 -0
- package/src/semantic/minhash.test.ts +197 -0
- package/src/semantic/minhash.ts +261 -0
- package/src/semantic/semantic-query.ts +173 -0
- package/src/semantic/semantic.test.ts +1315 -0
- package/src/semantic/similarity.ts +268 -0
- package/src/semantic/types.ts +145 -0
- package/src/semantic/vectors.ts +235 -0
package/src/row-types.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed-row helpers for better-sqlite3 — narrow the library's `unknown`
|
|
3
|
+
* return values with a runtime shape check before reading properties.
|
|
4
|
+
*
|
|
5
|
+
* better-sqlite3 returns `unknown` for `.get()` and `unknown[]` for
|
|
6
|
+
* `.all()`. The codebase has historically relied on inline `as` casts;
|
|
7
|
+
* this module provides a typed wrapper that validates the SHAPE at runtime
|
|
8
|
+
* so a bad query (column renames, dropped columns) fails loudly instead
|
|
9
|
+
* of silently reading `undefined`.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export type SqliteRow = Record<string, unknown>;
|
|
13
|
+
|
|
14
|
+
export function isSqliteRow(value: unknown): value is SqliteRow {
|
|
15
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Narrow a `.get()` result. Returns `undefined` when the row is missing
|
|
20
|
+
* OR when the expected columns are absent. Callers can then assert the
|
|
21
|
+
* expected shape (the columns are guaranteed to exist after this check).
|
|
22
|
+
*/
|
|
23
|
+
export function expectRow<T extends SqliteRow>(
|
|
24
|
+
value: unknown,
|
|
25
|
+
columns: readonly (keyof T)[],
|
|
26
|
+
): T | undefined {
|
|
27
|
+
if (!isSqliteRow(value)) return undefined;
|
|
28
|
+
for (const col of columns) {
|
|
29
|
+
if (!(col in value)) return undefined;
|
|
30
|
+
}
|
|
31
|
+
return value as T;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function expectRows<T extends SqliteRow>(
|
|
35
|
+
value: unknown,
|
|
36
|
+
columns: readonly (keyof T)[],
|
|
37
|
+
): T[] {
|
|
38
|
+
if (!Array.isArray(value)) return [];
|
|
39
|
+
const out: T[] = [];
|
|
40
|
+
for (const row of value) {
|
|
41
|
+
const narrowed = expectRow<T>(row, columns);
|
|
42
|
+
if (narrowed) out.push(narrowed);
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical-text builder tests (issue #1556 step 1 — prove-fail-before).
|
|
3
|
+
*
|
|
4
|
+
* Rule 23/38: ONE canonical form. Two formatting-variant fixtures of the
|
|
5
|
+
* same function MUST produce the same canonical text; the embedded string
|
|
6
|
+
* equals the hashed string (one assertion).
|
|
7
|
+
*
|
|
8
|
+
* Rule 37: cache invalidation. When canonical text changes (rename / body
|
|
9
|
+
* edit), the hash changes.
|
|
10
|
+
*/
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import test from "node:test";
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
buildCanonicalText,
|
|
16
|
+
buildCanonicalTextAndHash,
|
|
17
|
+
canonicalTextHash,
|
|
18
|
+
collapseWhitespace,
|
|
19
|
+
} from "./canonical-text.js";
|
|
20
|
+
import type { SymbolIR } from "@remnic/core";
|
|
21
|
+
|
|
22
|
+
function fn(qname: string, rawText: string): SymbolIR {
|
|
23
|
+
return {
|
|
24
|
+
kind: "function",
|
|
25
|
+
name: qname.split(".").pop() ?? qname,
|
|
26
|
+
qualifiedName: qname,
|
|
27
|
+
span: { startByte: 0, endByte: rawText.length },
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
32
|
+
// Rule 23/38: ONE canonical form — formatting variants hash identically.
|
|
33
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
34
|
+
|
|
35
|
+
test("canonical text: whitespace-only variants produce identical text", () => {
|
|
36
|
+
const variantA = `function add(a, b) {\n return a + b;\n}`;
|
|
37
|
+
const variantB = `function add(a,b){\n\treturn a+b;\n}`;
|
|
38
|
+
const sym = fn("mod.add", variantA);
|
|
39
|
+
const textA = buildCanonicalText({ symbol: sym, rawText: variantA });
|
|
40
|
+
const textB = buildCanonicalText({ symbol: sym, rawText: variantB });
|
|
41
|
+
assert.equal(textA, textB, "indentation/spacing variants must canonicalize identically");
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("canonical text: brace-style variants produce identical text", () => {
|
|
45
|
+
const sameLine = `function foo(x) { return x * 2; }`;
|
|
46
|
+
const newLine = `function foo(x)\n{\n return x * 2;\n}`;
|
|
47
|
+
const sym = fn("mod.foo", sameLine);
|
|
48
|
+
assert.equal(
|
|
49
|
+
buildCanonicalText({ symbol: sym, rawText: sameLine }),
|
|
50
|
+
buildCanonicalText({ symbol: sym, rawText: newLine }),
|
|
51
|
+
);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("canonical text: the embedded string equals the hashed string (rule 23)", () => {
|
|
55
|
+
const raw = `function bar(n) { return n + 1; }`;
|
|
56
|
+
const sym = fn("mod.bar", raw);
|
|
57
|
+
const { text, hash } = buildCanonicalTextAndHash({ symbol: sym, rawText: raw });
|
|
58
|
+
// The hash is over the EXACT text returned — no transformation gap.
|
|
59
|
+
assert.equal(hash, canonicalTextHash(text));
|
|
60
|
+
// And a different text produces a different hash.
|
|
61
|
+
const other = buildCanonicalTextAndHash({ symbol: fn("mod.baz", raw), rawText: raw });
|
|
62
|
+
assert.notEqual(hash, other.hash);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
66
|
+
// Rule 37: cache invalidation — rename / body change changes the hash.
|
|
67
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
68
|
+
|
|
69
|
+
test("canonical hash: rename changes the hash (cache invalidation)", () => {
|
|
70
|
+
const raw = `function f() { return 42; }`;
|
|
71
|
+
const before = buildCanonicalTextAndHash({ symbol: fn("mod.oldName", raw), rawText: raw });
|
|
72
|
+
const after = buildCanonicalTextAndHash({ symbol: fn("mod.newName", raw), rawText: raw });
|
|
73
|
+
assert.notEqual(before.hash, after.hash, "a rename must invalidate the cache");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("canonical hash: body change changes the hash", () => {
|
|
77
|
+
const rawA = `function f() { return 1; }`;
|
|
78
|
+
const rawB = `function f() { return 2; }`;
|
|
79
|
+
const before = buildCanonicalTextAndHash({ symbol: fn("mod.f", rawA), rawText: rawA });
|
|
80
|
+
const after = buildCanonicalTextAndHash({ symbol: fn("mod.f", rawB), rawText: rawB });
|
|
81
|
+
assert.notEqual(before.hash, after.hash);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("canonical hash: unchanged symbol is stable (idempotent)", () => {
|
|
85
|
+
const raw = `function stable() { console.log("hi"); }`;
|
|
86
|
+
const a = buildCanonicalTextAndHash({ symbol: fn("mod.stable", raw), rawText: raw });
|
|
87
|
+
const b = buildCanonicalTextAndHash({ symbol: fn("mod.stable", raw), rawText: raw });
|
|
88
|
+
assert.equal(a.hash, b.hash);
|
|
89
|
+
assert.equal(a.text, b.text);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
93
|
+
// collapseWhitespace unit
|
|
94
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
95
|
+
|
|
96
|
+
test("collapseWhitespace: tabs/newlines/multiple-spaces → single space", () => {
|
|
97
|
+
assert.equal(collapseWhitespace("\t\thello world\t"), "hello world");
|
|
98
|
+
assert.equal(collapseWhitespace("a\n\n\n\nb"), "a b");
|
|
99
|
+
assert.equal(collapseWhitespace(" line one \n line two "), "line one line two");
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
103
|
+
// Body truncation respects maxBodyLines
|
|
104
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
105
|
+
|
|
106
|
+
test("canonical text: body truncated to maxBodyLines (token budget)", () => {
|
|
107
|
+
const raw = `function big() { ${Array.from({ length: 50 }, (_, i) => `const x${i} = ${i};`).join(" ")} }`;
|
|
108
|
+
const sym = fn("mod.big", raw);
|
|
109
|
+
const text4 = buildCanonicalText({ symbol: sym, rawText: raw, maxBodyLines: 4 });
|
|
110
|
+
const textAll = buildCanonicalText({ symbol: sym, rawText: raw, maxBodyLines: 0 });
|
|
111
|
+
const body4 = text4.split("BODY:")[1]!.trim();
|
|
112
|
+
const bodyAll = textAll.split("BODY:")[1]!.trim();
|
|
113
|
+
assert.ok(body4.split(/\s+/).length <= 4, `truncated body should have <=4 tokens, got ${body4.split(/\s+/).length}`);
|
|
114
|
+
assert.ok(bodyAll.split(/\s+/).length > 4, "full body should have more than 4 tokens");
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
118
|
+
// Braceless arrow functions keep their body (cursor Bugbot: 'Arrow bodies
|
|
119
|
+
// lost in canonical text'). collapseWhitespace splits `=>` into `= >`, so
|
|
120
|
+
// the signature/body split must search the NORMALIZED arrow form.
|
|
121
|
+
// ──────────────────────────────────────────────────────────────────────────
|
|
122
|
+
|
|
123
|
+
test("canonical text: braceless arrow body is captured, not empty", () => {
|
|
124
|
+
const raw = "const double = (x) => x * 2";
|
|
125
|
+
const sym = fn("mod.double", raw);
|
|
126
|
+
const text = buildCanonicalText({ symbol: sym, rawText: raw });
|
|
127
|
+
const body = text.split("BODY:")[1]!.trim();
|
|
128
|
+
assert.ok(body.length > 0, `braceless arrow should keep a non-empty body, got "${body}"`);
|
|
129
|
+
assert.ok(body.includes("x"), `body should include the expression text, got "${body}"`);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("canonical text: braceless arrow variants canonicalize identically", () => {
|
|
133
|
+
const spaced = "const f = (x) => x + 1";
|
|
134
|
+
const tight = "const f=(x)=>x+1";
|
|
135
|
+
const sym = fn("mod.f", spaced);
|
|
136
|
+
assert.equal(
|
|
137
|
+
buildCanonicalText({ symbol: sym, rawText: spaced }),
|
|
138
|
+
buildCanonicalText({ symbol: sym, rawText: tight }),
|
|
139
|
+
);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
test("canonical text: braced arrow still splits at the brace", () => {
|
|
143
|
+
const raw = "const f = (x) => { return x + 1; }";
|
|
144
|
+
const sym = fn("mod.f", raw);
|
|
145
|
+
const text = buildCanonicalText({ symbol: sym, rawText: raw });
|
|
146
|
+
const body = text.split("BODY:")[1]!.trim();
|
|
147
|
+
assert.ok(body.length > 0, "braced arrow should keep a body");
|
|
148
|
+
assert.ok(body.includes("return"), `braced arrow body should include the block, got "${body}"`);
|
|
149
|
+
});
|
|
150
|
+
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical-text builder for symbol embeddings (issue #1556).
|
|
3
|
+
*
|
|
4
|
+
* Rule 23/38 — ONE canonical form. The exact string that is embedded is
|
|
5
|
+
* the exact string that is hashed for the embedding cache. Two formatting-
|
|
6
|
+
* variant fixtures of the same function MUST produce the same canonical
|
|
7
|
+
* text; the cache-hash test asserts this end to end.
|
|
8
|
+
*
|
|
9
|
+
* The canonical form (per the issue design):
|
|
10
|
+
* `signature + doc comment + first N lines of body`,
|
|
11
|
+
* normalized (sorted-key serialization where structured, stable truncation).
|
|
12
|
+
*
|
|
13
|
+
* Normalization strategy: ALL whitespace (including newlines) is collapsed
|
|
14
|
+
* to single spaces BEFORE signature/body split. This absorbs every
|
|
15
|
+
* formatting variant (indentation, brace placement, trailing whitespace,
|
|
16
|
+
* tabs vs spaces) so two semantically-identical functions produce IDENTICAL
|
|
17
|
+
* canonical text. The "first N lines" budget is then applied as a stable
|
|
18
|
+
* TOKEN budget over the collapsed text — stable because token count is
|
|
19
|
+
* formatting-independent.
|
|
20
|
+
*
|
|
21
|
+
* IMPORTANT: this module takes pre-extracted text spans, not raw source.
|
|
22
|
+
* The caller (the indexer) slices `[startByte, endByte)` from disk and
|
|
23
|
+
* passes the raw symbol text. Canonicalization here is about producing a
|
|
24
|
+
* stable embedding input from that raw text, not about re-parsing.
|
|
25
|
+
*/
|
|
26
|
+
import { createHash } from "node:crypto";
|
|
27
|
+
|
|
28
|
+
import type { SymbolIR } from "@remnic/core";
|
|
29
|
+
|
|
30
|
+
import { DEFAULT_CANONICAL_BODY_LINES } from "./config.js";
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Input to the canonical-text builder. `rawText` is the symbol's on-disk
|
|
34
|
+
* source slice `[startByte, endByte)`. `docComment` is the leading
|
|
35
|
+
* comment block immediately above the symbol, if the caller extracted one
|
|
36
|
+
* (the parser does not currently emit doc comments on SymbolIR, so this
|
|
37
|
+
* is optional and may be empty/undefined — the canonical form degrades
|
|
38
|
+
* gracefully to `signature + body`).
|
|
39
|
+
*/
|
|
40
|
+
export interface CanonicalTextInput {
|
|
41
|
+
readonly symbol: SymbolIR;
|
|
42
|
+
/** Raw source text of the symbol span. */
|
|
43
|
+
readonly rawText: string;
|
|
44
|
+
/** Optional leading doc comment (/** … *\/ or // … lines). */
|
|
45
|
+
readonly docComment?: string;
|
|
46
|
+
/** Body token budget (default {@link DEFAULT_CANONICAL_BODY_LINES}). */
|
|
47
|
+
readonly maxBodyLines?: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Collapse ALL whitespace runs (including newlines) to single spaces and
|
|
52
|
+
* trim. This is the universal normalization pass: it absorbs indentation,
|
|
53
|
+
* brace placement, trailing whitespace, tabs vs spaces, and line-ending
|
|
54
|
+
* differences. After this pass, two formatting variants of the same
|
|
55
|
+
* function are byte-identical.
|
|
56
|
+
*/
|
|
57
|
+
export function collapseWhitespace(text: string): string {
|
|
58
|
+
return text
|
|
59
|
+
// Insert spaces around punctuation so a,b and a, b canonicalize identically.
|
|
60
|
+
.replace(/([{}()<>\[\],;:?!=+\-*/%&|^~])/g, " $1 ")
|
|
61
|
+
// Collapse all whitespace runs (including those introduced above) to single spaces.
|
|
62
|
+
.replace(/\s+/g, " ")
|
|
63
|
+
.trim();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Line-oriented whitespace collapse (preserves line structure). Used when
|
|
68
|
+
* line boundaries matter (e.g. the "first N lines" budget needs real
|
|
69
|
+
* lines). Collapses intra-line whitespace runs but keeps newlines.
|
|
70
|
+
*/
|
|
71
|
+
export function collapseWhitespaceKeepLines(text: string): string {
|
|
72
|
+
return text
|
|
73
|
+
.split(/\r?\n/)
|
|
74
|
+
.map((line) => line.replace(/[ \t]+/g, " ").trim())
|
|
75
|
+
.join("\n")
|
|
76
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
77
|
+
.trim();
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Split normalized symbol text into signature + body at the first body-
|
|
82
|
+
* open marker (`{`, `=>`, `:` preceded by a non-type context). The split
|
|
83
|
+
* is CHARACTER-level on the fully-whitespace-normalized text, so a
|
|
84
|
+
* same-line function `function f() { return 1; }` and a brace-newline
|
|
85
|
+
* variant `function f()\n{ return 1; }` both split at the same `{`.
|
|
86
|
+
*
|
|
87
|
+
* Returns `{ signature, body }` — both whitespace-normalized. The
|
|
88
|
+
* signature includes the marker char so the canonical form is stable.
|
|
89
|
+
*/
|
|
90
|
+
function splitSignatureBody(normalized: string): { readonly signature: string; readonly body: string } {
|
|
91
|
+
// Find the body-open marker. `{` is the universal one (functions,
|
|
92
|
+
// classes, blocks). When there is no `{` we fall back to the arrow
|
|
93
|
+
// form so braceless arrow functions (`const f = (x) => x + 1`) split
|
|
94
|
+
// at the arrow instead of collapsing to an empty body.
|
|
95
|
+
//
|
|
96
|
+
// collapseWhitespace's punctuation-spacer turns `=>` into `= >`, so
|
|
97
|
+
// the raw `=>` marker NEVER appears in the normalized text — search
|
|
98
|
+
// the normalized form (`= >`) or this whole branch is dead and
|
|
99
|
+
// braceless arrows lose their body (cursor Bugbot: 'Arrow bodies lost
|
|
100
|
+
// in canonical text'). Embeddings, cache hashes, and MinHash inputs
|
|
101
|
+
// then omitted arrow-function logic.
|
|
102
|
+
const braceIdx = normalized.indexOf("{");
|
|
103
|
+
if (braceIdx >= 0) {
|
|
104
|
+
const signature = normalized.slice(0, braceIdx + 1).trim();
|
|
105
|
+
const body = normalized.slice(braceIdx + 1).trim();
|
|
106
|
+
return { signature, body };
|
|
107
|
+
}
|
|
108
|
+
const arrowIdx = normalized.indexOf("=>");
|
|
109
|
+
const arrowNormIdx = normalized.indexOf("= >");
|
|
110
|
+
if (arrowIdx >= 0) {
|
|
111
|
+
const signature = normalized.slice(0, arrowIdx + 2).trim();
|
|
112
|
+
const body = normalized.slice(arrowIdx + 2).trim();
|
|
113
|
+
return { signature, body };
|
|
114
|
+
}
|
|
115
|
+
if (arrowNormIdx >= 0) {
|
|
116
|
+
// `= >` spans 3 chars (`= `, ` `, `>`); cut after the `>`.
|
|
117
|
+
const bodyStart = arrowNormIdx + 3;
|
|
118
|
+
const signature = normalized.slice(0, bodyStart).trim();
|
|
119
|
+
const body = normalized.slice(bodyStart).trim();
|
|
120
|
+
return { signature, body };
|
|
121
|
+
}
|
|
122
|
+
// No body marker — treat the whole text as signature (e.g. `type Foo = string`).
|
|
123
|
+
return { signature: normalized, body: "" };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Extract a coarse signature string from raw symbol text. The text is
|
|
128
|
+
* fully whitespace-normalized first, then split at the first body-open
|
|
129
|
+
* marker. The returned signature is stable across all formatting variants
|
|
130
|
+
* of the same function.
|
|
131
|
+
*/
|
|
132
|
+
export function extractSignatureLine(
|
|
133
|
+
rawText: string,
|
|
134
|
+
_kind: SymbolIR["kind"],
|
|
135
|
+
): string {
|
|
136
|
+
const normalized = collapseWhitespace(rawText);
|
|
137
|
+
return splitSignatureBody(normalized).signature;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Extract the body (everything after the signature/header marker) from
|
|
142
|
+
* raw symbol text, truncated to the first `maxBodyLines` tokens. Tokens
|
|
143
|
+
* are whitespace-delimited words/operators in the normalized body text —
|
|
144
|
+
* this is a STABLE budget (formatting-independent) unlike line-based
|
|
145
|
+
* truncation. `maxBodyLines` is the config field name; it functions as a
|
|
146
|
+
* token budget here (each "line" ≈ one significant token).
|
|
147
|
+
*
|
|
148
|
+
* `maxBodyLines <= 0` means unlimited (return the full normalized body).
|
|
149
|
+
*/
|
|
150
|
+
export function extractBodyText(rawText: string, maxBodyLines: number): string {
|
|
151
|
+
const normalized = collapseWhitespace(rawText);
|
|
152
|
+
const body = splitSignatureBody(normalized).body;
|
|
153
|
+
if (maxBodyLines <= 0 || body.length === 0) return body;
|
|
154
|
+
const tokens = body.split(/\s+/);
|
|
155
|
+
return tokens.slice(0, maxBodyLines).join(" ");
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Build the canonical embedding text for a symbol.
|
|
160
|
+
*
|
|
161
|
+
* The form (rule 23/38 — ONE form, every consumer):
|
|
162
|
+
* KIND:<kind>\nQNAME:<qualifiedName>\nSIG:<signature>\n[DOC:<doc>]\nBODY:<body>
|
|
163
|
+
*
|
|
164
|
+
* `kind` and `qualifiedName` are included as stable prefix lines so two
|
|
165
|
+
* functions with identical bodies but different names (the "renamed
|
|
166
|
+
* variable" clone fixture) embed close but not identically — the qualified
|
|
167
|
+
* name differentiates them at the embedding level while the body dominates
|
|
168
|
+
* the similarity. The cache hash, by contrast, is over the FULL canonical
|
|
169
|
+
* text including the name, so a rename invalidates the cache (rule 37).
|
|
170
|
+
*
|
|
171
|
+
* Normalization:
|
|
172
|
+
* - ALL whitespace collapsed (indentation/brace-style/newlines absorbed)
|
|
173
|
+
* - body truncated to maxBodyLines tokens (stable budget)
|
|
174
|
+
*/
|
|
175
|
+
export function buildCanonicalText(input: CanonicalTextInput): string {
|
|
176
|
+
const { symbol, rawText, docComment, maxBodyLines } = input;
|
|
177
|
+
const budget = maxBodyLines ?? DEFAULT_CANONICAL_BODY_LINES;
|
|
178
|
+
const signature = extractSignatureLine(rawText, symbol.kind);
|
|
179
|
+
const body = extractBodyText(rawText, budget);
|
|
180
|
+
const doc = docComment ? collapseWhitespace(docComment) : "";
|
|
181
|
+
const parts = [
|
|
182
|
+
`KIND:${symbol.kind}`,
|
|
183
|
+
`QNAME:${symbol.qualifiedName}`,
|
|
184
|
+
`SIG:${signature}`,
|
|
185
|
+
];
|
|
186
|
+
if (doc.length > 0) parts.push(`DOC:${doc}`);
|
|
187
|
+
parts.push(`BODY:${body}`);
|
|
188
|
+
return parts.join("\n");
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* The cache key for a canonical text. This is sha256 over the EXACT
|
|
193
|
+
* canonical text string (rule 23 — the embedded string equals the hashed
|
|
194
|
+
* string). When the canonical text changes (e.g. a rename edits the
|
|
195
|
+
* qualified name, or the body changes), the hash changes, and:
|
|
196
|
+
* 1. the cached vector is invalidated (re-embedded), and
|
|
197
|
+
* 2. any SIMILAR_TO edge derived from it is recomputed.
|
|
198
|
+
*
|
|
199
|
+
* This is the single chokepoint for cache invalidation (rule 37). Every
|
|
200
|
+
* layer that persists a vector persists THIS hash alongside it; every
|
|
201
|
+
* re-index compares THIS hash to decide whether to re-embed.
|
|
202
|
+
*/
|
|
203
|
+
export function canonicalTextHash(canonicalText: string): string {
|
|
204
|
+
return createHash("sha256").update(canonicalText, "utf8").digest("hex");
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Convenience: build canonical text AND its hash in one call. The hash is
|
|
209
|
+
* over the returned text — callers that store both MUST store the exact
|
|
210
|
+
* `text` alongside the `hash` (never re-derive the text from disk and hash
|
|
211
|
+
* separately, or a formatter run between the two would silently
|
|
212
|
+
* re-embed — rule 37).
|
|
213
|
+
*/
|
|
214
|
+
export function buildCanonicalTextAndHash(
|
|
215
|
+
input: CanonicalTextInput,
|
|
216
|
+
): { readonly text: string; readonly hash: string } {
|
|
217
|
+
const text = buildCanonicalText(input);
|
|
218
|
+
return { text, hash: canonicalTextHash(text) };
|
|
219
|
+
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Semantic-layer configuration for @remnic/coding-graph (issue #1556).
|
|
3
|
+
*
|
|
4
|
+
* Rule 30/48: the semantic layer is OFF by default. Embedding costs compute
|
|
5
|
+
* and possibly tokens, so nothing in this module touches a provider, writes
|
|
6
|
+
* a vector, or sends symbol text off-machine unless `enabled` is explicitly
|
|
7
|
+
* true. The gate-off characterization test (`gate-off.test.ts`) asserts
|
|
8
|
+
* this end to end.
|
|
9
|
+
*
|
|
10
|
+
* The config object is intentionally self-contained in this package rather
|
|
11
|
+
* than wired into @remnic/core/config.ts — the coding-graph package is an
|
|
12
|
+
* optional peer dep and must compile standalone. Host integrations
|
|
13
|
+
* (core/config.ts + openclaw.plugin.json schema) resolve to this same
|
|
14
|
+
* shape via `resolveSemanticConfig()`, which reads the documented env vars
|
|
15
|
+
* with the `ENGRAM_` fallback (gotcha 9).
|
|
16
|
+
*/
|
|
17
|
+
import type { EdgeProvenance } from "../graph-schema.js";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Default SIMILAR_TO cosine confirmation threshold (issue #1556 design).
|
|
21
|
+
* 0.92 is the codebase-memory-mcp precedent — high enough to avoid
|
|
22
|
+
* false near-clone pairs across structurally-similar but logically-distinct
|
|
23
|
+
* functions, low enough to catch genuine copy-paste with a renamed
|
|
24
|
+
* variable.
|
|
25
|
+
*/
|
|
26
|
+
export const DEFAULT_SIMILAR_TO_THRESHOLD = 0.92;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Default maximum symbols embedded per indexing run. Bounds per-run
|
|
30
|
+
* provider cost. 0 means unlimited (the host budget is the only cap).
|
|
31
|
+
*/
|
|
32
|
+
export const DEFAULT_MAX_SYMBOLS_PER_RUN = 0;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Confidence band assigned to MinHash-only SIMILAR_TO edges when no
|
|
36
|
+
* embedding provider is available (deterministic, local). Kept below the
|
|
37
|
+
* embedding-confirmed band so consumers can distinguish provenance quality.
|
|
38
|
+
* The issue designates this a distinct, documented lower band.
|
|
39
|
+
*/
|
|
40
|
+
export const MINHASH_ONLY_CONFIDENCE = 0.5;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Confidence band assigned to embedding-confirmed SIMILAR_TO edges. Uses
|
|
44
|
+
* the actual cosine similarity score (≥ similarToThreshold), so this is
|
|
45
|
+
* the FLOOR for that band — the real confidence is the cosine value.
|
|
46
|
+
*/
|
|
47
|
+
export const EMBEDDING_CONFIRMED_MIN_CONFIDENCE = 0.92;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Edge type emitted by the SIMILAR_TO pipeline. Lives in the `edges`
|
|
51
|
+
* table with `provenance: "semantic"` (already in EDGE_PROVENANCE_VALUES).
|
|
52
|
+
*/
|
|
53
|
+
export const SIMILAR_TO_EDGE_TYPE = "SIMILAR_TO";
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The single provenance tag for every edge this module writes.
|
|
57
|
+
*/
|
|
58
|
+
export const SEMANTIC_PROVENANCE: EdgeProvenance = "semantic";
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Canonical-text body line budget. `signature + doc comment + first N lines
|
|
62
|
+
* of body` per the issue design. N is bounded so a 5k-line function does
|
|
63
|
+
* not dominate the embedded string (and the provider token budget).
|
|
64
|
+
*/
|
|
65
|
+
export const DEFAULT_CANONICAL_BODY_LINES = 16;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Token shingle width for MinHash. 3 tokens per shingle is the standard
|
|
69
|
+
* near-duplicate-detection width — small enough to catch renamed-variable
|
|
70
|
+
* clones (most shingles survive a single rename), large enough that
|
|
71
|
+
* boilerplate coincidence does not flood the candidate set.
|
|
72
|
+
*/
|
|
73
|
+
export const MINHASH_SHINGLE_WIDTH = 2;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Number of MinHash permutations (hash functions). More = tighter Jaccard
|
|
77
|
+
* estimate = more compute. 128 gives ~±5% Jaccard error at p=0.95 which
|
|
78
|
+
* is well inside the 0.92 cosine confirmation gate's margin.
|
|
79
|
+
*/
|
|
80
|
+
export const MINHASH_NUM_PERMUTATIONS = 128;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Number of LSH bands. More bands = more candidate pairs (higher recall,
|
|
84
|
+
* lower precision before the cosine gate). 32 bands of 4 rows each is the
|
|
85
|
+
* banding that pairs a Jaccard ≥ 0.4 with high probability while keeping
|
|
86
|
+
* the candidate set small for typical repos.
|
|
87
|
+
*/
|
|
88
|
+
export const LSH_NUM_BANDS = 32;
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The LSH banding — derived: rows per band = permutations / bands.
|
|
92
|
+
*/
|
|
93
|
+
export const LSH_ROWS_PER_BAND = MINHASH_NUM_PERMUTATIONS / LSH_NUM_BANDS;
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* LSH candidate-pair Jaccard floor implied by the banding (s ≈ (1/b)^(1/r)).
|
|
97
|
+
* Below this Jaccard similarity a pair is almost never a candidate. Kept as
|
|
98
|
+
* a named constant so the determinism test can assert the candidate set is
|
|
99
|
+
* a pure function of (seeds, inputs) without a hidden threshold drift.
|
|
100
|
+
*/
|
|
101
|
+
export const LSH_CANDIDATE_JACCARD_FLOOR = Math.pow(1 / LSH_NUM_BANDS, 1 / LSH_ROWS_PER_BAND);
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Self-contained semantic config. Resolved from host config + env.
|
|
105
|
+
*
|
|
106
|
+
* `enabled` is the single gate for the whole layer. When false, every
|
|
107
|
+
* semantic entry point (index-time vector writes, SIMILAR_TO edges,
|
|
108
|
+
* semantic_query) returns a tagged `{ ok: false, code: "semantic_disabled" }`
|
|
109
|
+
* WITHOUT touching the provider or the vectors table (gate-off parity).
|
|
110
|
+
*/
|
|
111
|
+
export interface SemanticConfig {
|
|
112
|
+
/** Master gate. Default false (rule 30/48). */
|
|
113
|
+
readonly enabled: boolean;
|
|
114
|
+
/** Cosine threshold for SIMILAR_TO confirmation. Default 0.92. */
|
|
115
|
+
readonly similarToThreshold: number;
|
|
116
|
+
/** Per-run embedding budget (0 = unlimited). Default 0. */
|
|
117
|
+
readonly maxSymbolsPerRun: number;
|
|
118
|
+
/** Canonical-text body line budget. Default 16. */
|
|
119
|
+
readonly canonicalBodyLines: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Environment variable names. `REMNIC_*` is primary; `ENGRAM_*` is the
|
|
124
|
+
* fallback (gotcha 9 — legacy env names still bind).
|
|
125
|
+
*/
|
|
126
|
+
const ENV_ENABLED = ["REMNIC_CODING_GRAPH_SEMANTIC_ENABLED", "ENGRAM_CODING_GRAPH_SEMANTIC_ENABLED"];
|
|
127
|
+
const ENV_THRESHOLD = ["REMNIC_CODING_GRAPH_SEMANTIC_SIMILAR_TO_THRESHOLD", "ENGRAM_CODING_GRAPH_SEMANTIC_SIMILAR_TO_THRESHOLD"];
|
|
128
|
+
const ENV_MAX_SYMBOLS = ["REMNIC_CODING_GRAPH_SEMANTIC_MAX_SYMBOLS_PER_RUN", "ENGRAM_CODING_GRAPH_SEMANTIC_MAX_SYMBOLS_PER_RUN"];
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Resolve a boolean env var. Accepts true/false/1/0 (case-insensitive).
|
|
132
|
+
*/
|
|
133
|
+
/**
|
|
134
|
+
* Coerce a host-provided boolean (which may arrive as a string/number from
|
|
135
|
+
* JSON or CLI config) to a real boolean, so "false"/"0"/"no" do not become
|
|
136
|
+
* truthy and silently enable vector indexing despite an explicit opt-out
|
|
137
|
+
* (chatgpt-codex-connector: 'Coerce host enabled before trusting it').
|
|
138
|
+
* undefined/null → undefined (fall through to env/default).
|
|
139
|
+
*/
|
|
140
|
+
function coerceHostBool(value: unknown): boolean | undefined {
|
|
141
|
+
if (value === undefined || value === null) return undefined;
|
|
142
|
+
if (typeof value === "boolean") return value;
|
|
143
|
+
if (typeof value === "string") {
|
|
144
|
+
const v = value.trim().toLowerCase();
|
|
145
|
+
// Fail CLOSED: only explicit affirmatives enable the layer. Any other
|
|
146
|
+
// value ("false"/"0"/"no"/"off"/"disabled"/""/unknown) is an opt-out,
|
|
147
|
+
// so a malformed or unrecognized host string can never silently enable
|
|
148
|
+
// remote embedding against operator intent (cursor Bugbot: 'Unknown
|
|
149
|
+
// enabled strings enable semantic').
|
|
150
|
+
if (v === "true" || v === "1" || v === "yes" || v === "on") return true;
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
if (typeof value === "number") return value !== 0;
|
|
154
|
+
return Boolean(value);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Coerce a host-provided number (which may arrive as a numeric string from
|
|
159
|
+
* JSON/CLI config) to a finite number, so a malformed value like
|
|
160
|
+
* maxSymbolsPerRun:"abc" cannot become NaN and silently disable the vector
|
|
161
|
+
* budget (NaN > 0 is false → unlimited) or break cosine confirmation
|
|
162
|
+
* (comparisons against NaN are false). undefined/null/non-finite → undefined
|
|
163
|
+
* (fall through to env/default). Negative finite numbers pass through so the
|
|
164
|
+
* downstream clamp still controls range (chatgpt-codex-connector: 'Validate
|
|
165
|
+
* host numeric config before clamping').
|
|
166
|
+
*/
|
|
167
|
+
function coerceHostNumber(value: unknown): number | undefined {
|
|
168
|
+
if (value === undefined || value === null) return undefined;
|
|
169
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : undefined;
|
|
170
|
+
if (typeof value === "string") {
|
|
171
|
+
const n = Number(value.trim());
|
|
172
|
+
return Number.isFinite(n) ? n : undefined;
|
|
173
|
+
}
|
|
174
|
+
return undefined;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function resolveBoolEnv(names: readonly string[], fallback: boolean, env: NodeJS.ProcessEnv): boolean {
|
|
178
|
+
for (const name of names) {
|
|
179
|
+
const raw = env[name];
|
|
180
|
+
if (raw === undefined) continue;
|
|
181
|
+
const v = raw.trim().toLowerCase();
|
|
182
|
+
if (v === "true" || v === "1") return true;
|
|
183
|
+
if (v === "false" || v === "0") return false;
|
|
184
|
+
// Malformed value: ignore (do not throw — a typo must not crash indexing).
|
|
185
|
+
}
|
|
186
|
+
return fallback;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Resolve a positive-number env var. Ignores malformed/NaN/negative values
|
|
191
|
+
* rather than throwing (a config typo must not crash the indexer).
|
|
192
|
+
*/
|
|
193
|
+
function resolveNumberEnv(names: readonly string[], fallback: number, env: NodeJS.ProcessEnv): number {
|
|
194
|
+
for (const name of names) {
|
|
195
|
+
const raw = env[name];
|
|
196
|
+
if (raw === undefined) continue;
|
|
197
|
+
const n = Number(raw);
|
|
198
|
+
if (Number.isFinite(n) && n >= 0) return n;
|
|
199
|
+
}
|
|
200
|
+
return fallback;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Resolve the semantic config from an optional host-provided partial plus
|
|
205
|
+
* the environment. Explicit host values win; env vars fill the gaps;
|
|
206
|
+
* documented defaults apply last.
|
|
207
|
+
*
|
|
208
|
+
* `env` defaults to `process.env` but is a parameter so tests can pin the
|
|
209
|
+
* environment deterministically (rule 38 — no implicit process state).
|
|
210
|
+
*/
|
|
211
|
+
export function resolveSemanticConfig(
|
|
212
|
+
host?: Partial<SemanticConfig>,
|
|
213
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
214
|
+
): SemanticConfig {
|
|
215
|
+
const enabled =
|
|
216
|
+
coerceHostBool(host?.enabled) ?? resolveBoolEnv(ENV_ENABLED, false, env);
|
|
217
|
+
const similarToThreshold =
|
|
218
|
+
coerceHostNumber(host?.similarToThreshold) ?? resolveNumberEnv(ENV_THRESHOLD, DEFAULT_SIMILAR_TO_THRESHOLD, env);
|
|
219
|
+
const maxSymbolsPerRun =
|
|
220
|
+
coerceHostNumber(host?.maxSymbolsPerRun) ?? resolveNumberEnv(ENV_MAX_SYMBOLS, DEFAULT_MAX_SYMBOLS_PER_RUN, env);
|
|
221
|
+
const canonicalBodyLines =
|
|
222
|
+
coerceHostNumber(host?.canonicalBodyLines) ?? DEFAULT_CANONICAL_BODY_LINES;
|
|
223
|
+
return {
|
|
224
|
+
enabled,
|
|
225
|
+
// Clamp threshold into [0,1] — a malformed env must not produce an
|
|
226
|
+
// out-of-range confidence gate.
|
|
227
|
+
similarToThreshold: Math.min(1, Math.max(0, similarToThreshold)),
|
|
228
|
+
maxSymbolsPerRun: Math.max(0, Math.floor(maxSymbolsPerRun)),
|
|
229
|
+
// canonicalBodyLines: a negative/zero value must NOT clamp to 0 because
|
|
230
|
+
// extractBodyText treats <= 0 as unlimited — sending full symbol bodies to
|
|
231
|
+
// the embedding provider instead of the bounded excerpt (defeats the
|
|
232
|
+
// privacy/cost cap). Fall back to the default instead (#1680).
|
|
233
|
+
canonicalBodyLines: canonicalBodyLines >= 1 ? Math.floor(canonicalBodyLines) : DEFAULT_CANONICAL_BODY_LINES,
|
|
234
|
+
};
|
|
235
|
+
}
|