@rigour-labs/core 6.5.1 → 6.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/context/automatic-index.d.ts +5 -0
- package/dist/context/automatic-index.js +24 -3
- package/dist/context/index-status.test.d.ts +1 -0
- package/dist/context/index-status.test.js +26 -0
- package/dist/deep/fact-extractor.js +2 -1
- package/dist/deep/fact-extractor.test.js +2 -1
- package/dist/gates/ast.js +4 -2
- package/dist/gates/coverage.js +4 -2
- package/dist/gates/dedupe-failures.test.d.ts +1 -0
- package/dist/gates/dedupe-failures.test.js +12 -0
- package/dist/gates/deprecated-dependencies.d.ts +27 -0
- package/dist/gates/deprecated-dependencies.js +104 -0
- package/dist/gates/deprecated-dependencies.test.d.ts +1 -0
- package/dist/gates/deprecated-dependencies.test.js +55 -0
- package/dist/gates/runner.d.ts +7 -1
- package/dist/gates/runner.js +25 -11
- package/dist/gates/security-patterns-data.js +2 -1
- package/dist/gates/security-patterns-owasp.test.js +15 -0
- package/dist/gates/semantic-bugs.js +1 -1
- package/dist/gates/unindexed-reads/coverage.d.ts +15 -0
- package/dist/gates/unindexed-reads/coverage.js +24 -0
- package/dist/gates/unindexed-reads/index.d.ts +20 -0
- package/dist/gates/unindexed-reads/index.js +98 -0
- package/dist/gates/unindexed-reads/queries.d.ts +17 -0
- package/dist/gates/unindexed-reads/queries.js +147 -0
- package/dist/gates/unindexed-reads/queries.test.d.ts +1 -0
- package/dist/gates/unindexed-reads/queries.test.js +54 -0
- package/dist/gates/unindexed-reads/schema.d.ts +38 -0
- package/dist/gates/unindexed-reads/schema.js +256 -0
- package/dist/gates/unindexed-reads/schema.test.d.ts +1 -0
- package/dist/gates/unindexed-reads/schema.test.js +81 -0
- package/dist/gates/unindexed-reads/sql.d.ts +21 -0
- package/dist/gates/unindexed-reads/sql.js +131 -0
- package/dist/gates/unindexed-reads/unindexed-reads.test.d.ts +1 -0
- package/dist/gates/unindexed-reads/unindexed-reads.test.js +90 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +4 -1
- package/dist/inference/brain-placeholder.test.d.ts +1 -0
- package/dist/inference/brain-placeholder.test.js +28 -0
- package/dist/inference/llama-engine-checksum.test.d.ts +1 -0
- package/dist/inference/llama-engine-checksum.test.js +27 -0
- package/dist/inference/llama-engine.d.ts +4 -1
- package/dist/inference/llama-engine.js +15 -8
- package/dist/memory/recall.d.ts +37 -0
- package/dist/memory/recall.js +93 -0
- package/dist/memory/recall.test.d.ts +1 -0
- package/dist/memory/recall.test.js +36 -0
- package/dist/pattern-index/assess.d.ts +16 -0
- package/dist/pattern-index/assess.js +37 -0
- package/dist/pattern-index/embeddings.d.ts +19 -0
- package/dist/pattern-index/embeddings.js +16 -0
- package/dist/pattern-index/index.d.ts +3 -2
- package/dist/pattern-index/index.js +4 -2
- package/dist/pattern-index/indexer.js +19 -9
- package/dist/pattern-index/matcher.d.ts +6 -0
- package/dist/pattern-index/matcher.js +13 -3
- package/dist/pattern-index/pattern-reuse.test.d.ts +1 -0
- package/dist/pattern-index/pattern-reuse.test.js +76 -0
- package/dist/pattern-index/staleness.d.ts +2 -15
- package/dist/pattern-index/staleness.js +1 -78
- package/dist/pattern-index/types.d.ts +2 -0
- package/dist/review/agent-fixes.d.ts +4 -0
- package/dist/review/agent-fixes.js +10 -7
- package/dist/review/agent-fixes.test.js +4 -4
- package/dist/review/changed-lines.d.ts +2 -1
- package/dist/review/changed-lines.js +12 -2
- package/dist/review/check-outcomes.d.ts +31 -0
- package/dist/review/check-outcomes.js +81 -0
- package/dist/review/check-outcomes.test.d.ts +1 -0
- package/dist/review/check-outcomes.test.js +56 -0
- package/dist/review/migration-order.d.ts +5 -0
- package/dist/review/migration-order.js +70 -0
- package/dist/review/migration-order.test.d.ts +1 -0
- package/dist/review/migration-order.test.js +62 -0
- package/dist/review/quiet.d.ts +2 -0
- package/dist/review/quiet.js +12 -3
- package/dist/review/review.d.ts +4 -0
- package/dist/review/review.js +17 -5
- package/dist/review/review.test.js +36 -1
- package/dist/semantic/ast.d.ts +1 -0
- package/dist/semantic/ast.js +4 -0
- package/dist/semantic/learn/compile.js +3 -1
- package/dist/semantic/learn/learn.js +40 -5
- package/dist/semantic/types.d.ts +2 -0
- package/dist/services/context-scope-summary.test.d.ts +1 -0
- package/dist/services/context-scope-summary.test.js +17 -0
- package/dist/services/context-telemetry-service.js +2 -1
- package/dist/storage/context-telemetry.js +33 -19
- package/dist/storage/db.js +52 -1
- package/dist/storage/findings.js +1 -1
- package/dist/storage/fix-lessons.d.ts +13 -0
- package/dist/storage/fix-lessons.js +84 -0
- package/dist/storage/fix-lessons.test.d.ts +1 -0
- package/dist/storage/fix-lessons.test.js +61 -0
- package/dist/storage/index.d.ts +2 -1
- package/dist/storage/index.js +2 -1
- package/dist/storage/lessons.d.ts +16 -2
- package/dist/storage/lessons.js +27 -61
- package/dist/storage/lessons.test.js +1 -15
- package/dist/storage/local-memory.js +39 -5
- package/dist/storage/local-memory.test.d.ts +1 -0
- package/dist/storage/local-memory.test.js +55 -0
- package/dist/storage/patterns.d.ts +1 -1
- package/dist/storage/patterns.js +1 -1
- package/dist/storage/share-memory.test.d.ts +1 -0
- package/dist/storage/share-memory.test.js +34 -0
- package/dist/storage/team-vector-store.js +2 -1
- package/dist/storage/telemetry-scope.test.d.ts +1 -0
- package/dist/storage/telemetry-scope.test.js +38 -0
- package/dist/templates/universal-config.js +11 -0
- package/dist/types/index.d.ts +132 -0
- package/dist/types/index.js +15 -0
- package/dist/utils/glob-paths.d.ts +4 -0
- package/dist/utils/glob-paths.js +26 -0
- package/dist/utils/glob-paths.test.d.ts +1 -0
- package/dist/utils/glob-paths.test.js +40 -0
- package/dist/utils/scanner.js +4 -2
- package/dist/utils/scanner.test.js +2 -1
- package/package.json +7 -7
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { policyColumns, policyFilters, replayMigrations } from './schema.js';
|
|
3
|
+
import { splitStatements } from './sql.js';
|
|
4
|
+
describe('splitStatements', () => {
|
|
5
|
+
it('keeps comments, quotes and dollar bodies from splitting a statement', () => {
|
|
6
|
+
const sql = `-- a; comment\ncreate table a (x text default ';');\n/* b; */ do $$ begin perform 1; end $$;\nselect 'it''s; fine';`;
|
|
7
|
+
expect(splitStatements(sql)).toEqual(["create table a (x text default ';')", 'do $$ begin perform 1; end $$', "select 'it''s; fine'"]);
|
|
8
|
+
});
|
|
9
|
+
});
|
|
10
|
+
describe('replayMigrations', () => {
|
|
11
|
+
it('collects primary keys, unique columns and table constraints as indexes', () => {
|
|
12
|
+
const schema = replayMigrations([`
|
|
13
|
+
create table public.orders (
|
|
14
|
+
id uuid primary key default gen_random_uuid(),
|
|
15
|
+
code text not null unique,
|
|
16
|
+
customer_id uuid not null references customers(id),
|
|
17
|
+
placed_at timestamptz,
|
|
18
|
+
constraint orders_customer_code unique (customer_id, code)
|
|
19
|
+
);`]);
|
|
20
|
+
expect(schema.get('orders').indexes.map(i => i.leading)).toEqual(['id', 'code', 'customer_id']);
|
|
21
|
+
expect(schema.get('orders').uncertain).toBe(false);
|
|
22
|
+
});
|
|
23
|
+
it('applies indexes, partial predicates, drops and renames in order', () => {
|
|
24
|
+
const schema = replayMigrations([
|
|
25
|
+
'create table orders (id uuid primary key, status text, placed_at timestamptz, customer_id uuid);',
|
|
26
|
+
`create index concurrently orders_open_idx on public.orders (placed_at desc, id) where status = 'open' and customer_id is not null;`,
|
|
27
|
+
'create index orders_customer_idx on orders using btree (customer_id);',
|
|
28
|
+
'create index orders_lower_idx on orders (lower(status));',
|
|
29
|
+
'drop index if exists public.orders_customer_idx;',
|
|
30
|
+
'alter index orders_open_idx rename to orders_open_v2_idx;',
|
|
31
|
+
]);
|
|
32
|
+
const indexes = schema.get('orders').indexes;
|
|
33
|
+
expect(indexes.map(i => [i.name, i.leading])).toEqual([[undefined, 'id'], ['orders_open_v2_idx', 'placed_at'], ['orders_lower_idx', null]]);
|
|
34
|
+
expect(indexes[1].predicate).toEqual([{ kind: 'eq', column: 'status', value: 'open' }, { kind: 'notnull', column: 'customer_id' }]);
|
|
35
|
+
});
|
|
36
|
+
it('follows ALTER TABLE: added keys, table renames, dropped tables', () => {
|
|
37
|
+
const schema = replayMigrations([
|
|
38
|
+
'create table events (id bigint, kind text);',
|
|
39
|
+
'alter table events add constraint events_pkey primary key (id);',
|
|
40
|
+
'alter table events add column ref text unique;',
|
|
41
|
+
'alter table events rename to audit_events;',
|
|
42
|
+
'create table scratch (x int);',
|
|
43
|
+
'drop table scratch;',
|
|
44
|
+
]);
|
|
45
|
+
expect(schema.get('audit_events').indexes.map(i => i.leading)).toEqual(['id', 'ref']);
|
|
46
|
+
expect([...schema.keys()]).toEqual(['audit_events']);
|
|
47
|
+
});
|
|
48
|
+
it('marks a table uncertain when DDL is beyond what it models', () => {
|
|
49
|
+
const schema = replayMigrations([
|
|
50
|
+
'create table a (id int, x int); create table b (id int); create table c (like a including indexes); create table d (id int) partition by range (id);',
|
|
51
|
+
'alter table a rename column x to y;',
|
|
52
|
+
`create or replace function f() returns void language plpgsql as $$ begin create index on b (id); end $$;`,
|
|
53
|
+
]);
|
|
54
|
+
expect(['a', 'b', 'c', 'd'].map(t => schema.get(t).uncertain)).toEqual([true, true, true, true]);
|
|
55
|
+
});
|
|
56
|
+
it('treats an OR or an unreadable predicate as unknown', () => {
|
|
57
|
+
const schema = replayMigrations([
|
|
58
|
+
'create table t (a int, b int);',
|
|
59
|
+
`create index t_or on t (a) where a = 1 or b = 2;`,
|
|
60
|
+
`create index t_fn on t (a) where coalesce(b, 0) > 0;`,
|
|
61
|
+
]);
|
|
62
|
+
expect(schema.get('t').indexes.map(i => i.predicate)).toEqual([[{ kind: 'unknown' }], [{ kind: 'unknown' }]]);
|
|
63
|
+
});
|
|
64
|
+
it('keeps SELECT and ALL policies as the columns they compare to the user', () => {
|
|
65
|
+
const schema = replayMigrations([
|
|
66
|
+
'create table subs (id text primary key, user_id uuid, status text);',
|
|
67
|
+
`create policy "own" on subs for select using (auth.uid() = user_id);`,
|
|
68
|
+
`create policy "writes" on subs for insert with check (auth.uid() = user_id);`,
|
|
69
|
+
`create policy "all" on subs to authenticated using (true);`,
|
|
70
|
+
]);
|
|
71
|
+
expect(policyFilters(schema.get('subs'))).toEqual({ columns: ['user_id'], unknown: false });
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
describe('policyColumns', () => {
|
|
75
|
+
it('reads comparisons to the current user and gives up on anything else', () => {
|
|
76
|
+
expect(policyColumns('(select auth.uid()) = owner_id')).toEqual(['owner_id']);
|
|
77
|
+
expect(policyColumns('(owner_id = auth.uid()) or (editor_id = auth.uid())')).toEqual(['owner_id', 'editor_id']);
|
|
78
|
+
expect(policyColumns('true')).toEqual([]);
|
|
79
|
+
expect(policyColumns('exists (select 1 from members m where m.org_id = org_id)')).toBeNull();
|
|
80
|
+
});
|
|
81
|
+
});
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Just enough SQL lexing to replay Postgres migrations: statements, top-level
|
|
3
|
+
* lists, and identifiers. Anything this cannot read is reported as unknown so
|
|
4
|
+
* the gate stays silent rather than guess.
|
|
5
|
+
*/
|
|
6
|
+
/** Statements in order, comments removed; a `$tag$ … $tag$` body stays inside its statement. */
|
|
7
|
+
export declare function splitStatements(sql: string): string[];
|
|
8
|
+
/** Split on `separator` outside parentheses and quotes. */
|
|
9
|
+
export declare function splitTopLevel(text: string, separator: RegExp): string[];
|
|
10
|
+
/** The text inside the parentheses that open at `open`, or undefined when they never close. */
|
|
11
|
+
export declare function parenthesized(text: string, open: number): {
|
|
12
|
+
inner: string;
|
|
13
|
+
end: number;
|
|
14
|
+
} | undefined;
|
|
15
|
+
/** Strip parentheses that enclose the whole expression, as often as they do: `((a))` → `a`, but `(a) or (b)` stays. */
|
|
16
|
+
export declare function unwrap(text: string): string;
|
|
17
|
+
/** A relation name as Rigour keys it: unquoted names fold to lower case, `public.` is implied. */
|
|
18
|
+
export declare function relationName(raw: string): string;
|
|
19
|
+
export declare function identifier(raw: string): string;
|
|
20
|
+
/** One identifier, optionally schema-qualified and quoted: the `name` in `CREATE TABLE name (`. */
|
|
21
|
+
export declare const NAME: string;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Just enough SQL lexing to replay Postgres migrations: statements, top-level
|
|
3
|
+
* lists, and identifiers. Anything this cannot read is reported as unknown so
|
|
4
|
+
* the gate stays silent rather than guess.
|
|
5
|
+
*/
|
|
6
|
+
/** Statements in order, comments removed; a `$tag$ … $tag$` body stays inside its statement. */
|
|
7
|
+
export function splitStatements(sql) {
|
|
8
|
+
const statements = [];
|
|
9
|
+
let current = '';
|
|
10
|
+
let i = 0;
|
|
11
|
+
while (i < sql.length) {
|
|
12
|
+
const rest = sql.slice(i);
|
|
13
|
+
const dollar = /^\$[A-Za-z_]*\$/.exec(rest);
|
|
14
|
+
if (rest.startsWith('--')) {
|
|
15
|
+
const end = sql.indexOf('\n', i);
|
|
16
|
+
i = end === -1 ? sql.length : end;
|
|
17
|
+
}
|
|
18
|
+
else if (rest.startsWith('/*')) {
|
|
19
|
+
const end = sql.indexOf('*/', i + 2);
|
|
20
|
+
i = end === -1 ? sql.length : end + 2;
|
|
21
|
+
current += ' ';
|
|
22
|
+
}
|
|
23
|
+
else if (dollar) {
|
|
24
|
+
const end = sql.indexOf(dollar[0], i + dollar[0].length);
|
|
25
|
+
const stop = end === -1 ? sql.length : end + dollar[0].length;
|
|
26
|
+
current += sql.slice(i, stop);
|
|
27
|
+
i = stop;
|
|
28
|
+
}
|
|
29
|
+
else if (sql[i] === "'" || sql[i] === '"') {
|
|
30
|
+
const stop = quotedEnd(sql, i);
|
|
31
|
+
current += sql.slice(i, stop);
|
|
32
|
+
i = stop;
|
|
33
|
+
}
|
|
34
|
+
else if (sql[i] === ';') {
|
|
35
|
+
if (current.trim())
|
|
36
|
+
statements.push(current.trim());
|
|
37
|
+
current = '';
|
|
38
|
+
i++;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
current += sql[i++];
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (current.trim())
|
|
45
|
+
statements.push(current.trim());
|
|
46
|
+
return statements.map(s => s.replace(/\s+/g, ' '));
|
|
47
|
+
}
|
|
48
|
+
/** Index just past a quoted string or identifier starting at `start` (a doubled quote is an escape). */
|
|
49
|
+
function quotedEnd(text, start) {
|
|
50
|
+
const quote = text[start];
|
|
51
|
+
let i = start + 1;
|
|
52
|
+
while (i < text.length) {
|
|
53
|
+
if (text[i] === quote && text[i + 1] === quote)
|
|
54
|
+
i += 2;
|
|
55
|
+
else if (text[i] === quote)
|
|
56
|
+
return i + 1;
|
|
57
|
+
else
|
|
58
|
+
i++;
|
|
59
|
+
}
|
|
60
|
+
return text.length;
|
|
61
|
+
}
|
|
62
|
+
/** Split on `separator` outside parentheses and quotes. */
|
|
63
|
+
export function splitTopLevel(text, separator) {
|
|
64
|
+
const parts = [];
|
|
65
|
+
let depth = 0;
|
|
66
|
+
let start = 0;
|
|
67
|
+
let i = 0;
|
|
68
|
+
while (i < text.length) {
|
|
69
|
+
const ch = text[i];
|
|
70
|
+
if (ch === "'" || ch === '"') {
|
|
71
|
+
i = quotedEnd(text, i);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (ch === '(')
|
|
75
|
+
depth++;
|
|
76
|
+
else if (ch === ')')
|
|
77
|
+
depth--;
|
|
78
|
+
else if (depth === 0) {
|
|
79
|
+
const match = separator.exec(text.slice(i));
|
|
80
|
+
if (match && match.index === 0) {
|
|
81
|
+
parts.push(text.slice(start, i).trim());
|
|
82
|
+
i += match[0].length;
|
|
83
|
+
start = i;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
i++;
|
|
88
|
+
}
|
|
89
|
+
parts.push(text.slice(start).trim());
|
|
90
|
+
return parts.filter(Boolean);
|
|
91
|
+
}
|
|
92
|
+
/** The text inside the parentheses that open at `open`, or undefined when they never close. */
|
|
93
|
+
export function parenthesized(text, open) {
|
|
94
|
+
let depth = 0;
|
|
95
|
+
for (let i = open; i < text.length; i++) {
|
|
96
|
+
const ch = text[i];
|
|
97
|
+
if (ch === "'" || ch === '"') {
|
|
98
|
+
i = quotedEnd(text, i) - 1;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (ch === '(')
|
|
102
|
+
depth++;
|
|
103
|
+
else if (ch === ')' && --depth === 0)
|
|
104
|
+
return { inner: text.slice(open + 1, i), end: i + 1 };
|
|
105
|
+
}
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
/** Strip parentheses that enclose the whole expression, as often as they do: `((a))` → `a`, but `(a) or (b)` stays. */
|
|
109
|
+
export function unwrap(text) {
|
|
110
|
+
let current = text.trim();
|
|
111
|
+
while (current.startsWith('(')) {
|
|
112
|
+
const group = parenthesized(current, 0);
|
|
113
|
+
if (!group || group.end !== current.length)
|
|
114
|
+
break;
|
|
115
|
+
current = group.inner.trim();
|
|
116
|
+
}
|
|
117
|
+
return current;
|
|
118
|
+
}
|
|
119
|
+
/** A relation name as Rigour keys it: unquoted names fold to lower case, `public.` is implied. */
|
|
120
|
+
export function relationName(raw) {
|
|
121
|
+
const parts = raw.split('.').map(identifier);
|
|
122
|
+
if (parts.length === 2 && parts[0] === 'public')
|
|
123
|
+
return parts[1];
|
|
124
|
+
return parts.join('.');
|
|
125
|
+
}
|
|
126
|
+
export function identifier(raw) {
|
|
127
|
+
const trimmed = raw.trim();
|
|
128
|
+
return trimmed.startsWith('"') ? trimmed.slice(1, -1).replace(/""/g, '"') : trimmed.toLowerCase();
|
|
129
|
+
}
|
|
130
|
+
/** One identifier, optionally schema-qualified and quoted: the `name` in `CREATE TABLE name (`. */
|
|
131
|
+
export const NAME = String.raw `(?:"(?:[^"]|"")+"|[A-Za-z_][\w$]*)(?:\.(?:"(?:[^"]|"")+"|[A-Za-z_][\w$]*))?`;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import os from 'os';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
5
|
+
import { UnindexedReadsGate } from './index.js';
|
|
6
|
+
let repo;
|
|
7
|
+
const write = (rel, body) => {
|
|
8
|
+
fs.mkdirSync(path.dirname(path.join(repo, rel)), { recursive: true });
|
|
9
|
+
fs.writeFileSync(path.join(repo, rel), body);
|
|
10
|
+
};
|
|
11
|
+
const run = (config = { enabled: true }) => new UnindexedReadsGate(config).run({ cwd: repo, patterns: ['src/**/*.ts'] });
|
|
12
|
+
const TABLE = `create table orders (
|
|
13
|
+
id uuid primary key,
|
|
14
|
+
customer_id uuid,
|
|
15
|
+
status text not null,
|
|
16
|
+
placed_at timestamptz,
|
|
17
|
+
campaign_id uuid
|
|
18
|
+
);
|
|
19
|
+
create index orders_customer_idx on orders (customer_id, status, placed_at desc);
|
|
20
|
+
create index orders_campaign_window_idx on orders (placed_at, id) where status = 'open' and campaign_id is not null;
|
|
21
|
+
`;
|
|
22
|
+
const WINDOW_READ = `
|
|
23
|
+
export async function openOrders(db: Db, since: string, until: string) {
|
|
24
|
+
return db
|
|
25
|
+
.from('orders')
|
|
26
|
+
.select('id, placed_at')
|
|
27
|
+
.eq('status', 'open')
|
|
28
|
+
.not('customer_id', 'is', null)
|
|
29
|
+
.gt('placed_at', since)
|
|
30
|
+
.lte('placed_at', until)
|
|
31
|
+
.order('placed_at')
|
|
32
|
+
.order('id');
|
|
33
|
+
}
|
|
34
|
+
`;
|
|
35
|
+
beforeEach(() => { repo = fs.mkdtempSync(path.join(os.tmpdir(), 'unindexed-')); });
|
|
36
|
+
afterEach(() => fs.rmSync(repo, { recursive: true, force: true }));
|
|
37
|
+
describe('UnindexedReadsGate', () => {
|
|
38
|
+
it('flags a window read whose only candidate index is partial on a filter the read does not have', async () => {
|
|
39
|
+
write('supabase/migrations/20260101000000_orders.sql', TABLE);
|
|
40
|
+
write('src/orders.ts', WINDOW_READ);
|
|
41
|
+
const failures = await run();
|
|
42
|
+
expect(failures).toHaveLength(1);
|
|
43
|
+
expect(failures[0]).toMatchObject({ id: 'unindexed-reads', files: ['src/orders.ts'], line: 4 });
|
|
44
|
+
expect(failures[0].details).toContain('`orders`');
|
|
45
|
+
expect(failures[0].details).toContain('`placed_at`');
|
|
46
|
+
});
|
|
47
|
+
it('is silent once a later migration adds an index that serves the read', async () => {
|
|
48
|
+
write('supabase/migrations/20260101000000_orders.sql', TABLE);
|
|
49
|
+
write('supabase/migrations/20260102000000_orders_open.sql', `create index concurrently orders_open_window_idx on public.orders (placed_at, id) where status = 'open' and customer_id is not null;`);
|
|
50
|
+
write('src/orders.ts', WINDOW_READ);
|
|
51
|
+
expect(await run()).toEqual([]);
|
|
52
|
+
});
|
|
53
|
+
it('is silent for a primary-key lookup, a table outside the migrations, and an uncertain table', async () => {
|
|
54
|
+
write('db/migrations/001_orders.sql', TABLE + 'create table notes (id int, body text);\nalter table notes rename column body to text;\n');
|
|
55
|
+
write('src/reads.ts', `
|
|
56
|
+
db.from('orders').select('*').eq('id', id);
|
|
57
|
+
db.from('auth_users').select('*').eq('email', email);
|
|
58
|
+
db.from('notes').select('*').eq('text', q);
|
|
59
|
+
`);
|
|
60
|
+
expect(await run()).toEqual([]);
|
|
61
|
+
});
|
|
62
|
+
it('is silent unless enabled, and for a read that narrows nothing', async () => {
|
|
63
|
+
write('supabase/migrations/20260101000000_orders.sql', TABLE);
|
|
64
|
+
write('src/orders.ts', WINDOW_READ + `\ndb.from('orders').select('*').limit(10);\n`);
|
|
65
|
+
expect(await run({ enabled: false })).toEqual([]);
|
|
66
|
+
expect(await new UnindexedReadsGate({ enabled: true }).run({ cwd: repo, patterns: ['src/**/*.ts'] })).toHaveLength(1);
|
|
67
|
+
});
|
|
68
|
+
it('needs every migration set that defines the table to lack a serving index', async () => {
|
|
69
|
+
write('apps/a/supabase/migrations/1_orders.sql', TABLE);
|
|
70
|
+
write('apps/b/supabase/migrations/1_orders.sql', TABLE + `create index orders_status_idx on orders (status);`);
|
|
71
|
+
write('src/orders.ts', WINDOW_READ);
|
|
72
|
+
expect(await run()).toEqual([]);
|
|
73
|
+
});
|
|
74
|
+
it('counts a row-level policy as a filter: names its column, and is silent once that column is indexed', async () => {
|
|
75
|
+
const subs = `create table subs (id text primary key, user_id uuid not null, status text);
|
|
76
|
+
create policy "own subs" on subs for select using (auth.uid() = user_id);`;
|
|
77
|
+
write('supabase/migrations/1_subs.sql', subs);
|
|
78
|
+
write('src/subs.ts', `export const active = (db: Db) => db.from('subs').select('*').in('status', ['active']).maybeSingle();\n`);
|
|
79
|
+
const [finding] = await run();
|
|
80
|
+
expect(finding.details).toContain('`user_id`');
|
|
81
|
+
write('supabase/migrations/2_subs_user.sql', 'create index subs_user_idx on subs (user_id);');
|
|
82
|
+
expect(await run()).toEqual([]);
|
|
83
|
+
});
|
|
84
|
+
it('is silent for a table whose policy it cannot read', async () => {
|
|
85
|
+
write('supabase/migrations/1_docs.sql', `create table docs (id int primary key, org_id int, title text);
|
|
86
|
+
create policy "members" on docs for select using (exists (select 1 from members m where m.org_id = docs.org_id));`);
|
|
87
|
+
write('src/docs.ts', `db.from('docs').select('*').eq('title', t);\n`);
|
|
88
|
+
expect(await run()).toEqual([]);
|
|
89
|
+
});
|
|
90
|
+
});
|
package/dist/index.d.ts
CHANGED
|
@@ -27,6 +27,7 @@ export { readLessons, writeLessons, promoteLesson, matchLessons, lessonText, les
|
|
|
27
27
|
export { lessonsForDiff, lessonsSection, type LessonMode } from './review-learning/team-lessons.js';
|
|
28
28
|
export { readRepoRules, rulesForDiff, rulesSection, splitRules, type RepoRule } from './review-learning/repo-rules.js';
|
|
29
29
|
export { diffFromGit, GitDiffError, type DiffSource } from './review/git-diff.js';
|
|
30
|
+
export { checkPrecisions, checkId, precisionOf, isMuted, MUTE_MIN_OUTCOMES, MUTE_BELOW, type CheckPrecision, type CheckOutcome } from './review/check-outcomes.js';
|
|
30
31
|
export { recordReviewOutcome, listResolvedFixes, removeResolvedFix, openFindingCount, type ResolvedFix, type FixCapture } from './review/agent-fixes.js';
|
|
31
32
|
export { computeEffectiveness, readAgentEvents, appendAgentEvent, findingKeys, type AgentEvent, type ReviewEffectiveness } from './review/effectiveness.js';
|
|
32
33
|
export { parseDiff, changedLinesByFile } from './utils/diff.js';
|
|
@@ -47,7 +48,9 @@ export { openDatabase, isSQLiteAvailable, compactDatabase, getDatabaseSize, rese
|
|
|
47
48
|
export type { RigourDB, CompactResult } from './storage/index.js';
|
|
48
49
|
export { checkLocalPatterns, persistAndReinforce, getProjectStats } from './storage/index.js';
|
|
49
50
|
export type { ProjectStats } from './storage/index.js';
|
|
50
|
-
export {
|
|
51
|
+
export { recordFixLessons, fixLessonSubject, fixLessonPrefix, FILES_TO_VALIDATE } from './storage/index.js';
|
|
52
|
+
export { recordInteractionEvidence, countInteractionEvidence, listLessons, listKnowledgeLessons, getApplicableLessons, transitionLesson, getRepositoryId, shareMemoryLesson, queueLocalLessonsForTeam } from './storage/index.js';
|
|
53
|
+
export { rankMemories, rankByMeaning, memoryText, SEMANTIC_FLOOR, RECALL_LIMIT, type MemoryEntry, type MemoryScope, type RankedMemory, type Ranked } from './memory/recall.js';
|
|
51
54
|
export type { ApplicableLessons, InteractionEvidence, LessonRecord, LessonState, LessonVisibility, LocalLessonImportOptions, LocalLessonImportResult } from './storage/index.js';
|
|
52
55
|
export { loadTeamConfiguration, saveTeamConfiguration, initializeTeamSchema, getTeamModeStatus, doctorTeamConnection, syncTeamOutbox, searchTeamKnowledge, backfillTeamEmbeddings, TEAM_CONFIG_PATH, validateTeamDatabaseUrl, explainTeamConnectionError, } from './storage/index.js';
|
|
53
56
|
export type { TeamConfiguration, TeamSemanticConfiguration, TeamModeStatus, TeamDoctorResult, SemanticKnowledgeCandidate, SemanticKnowledgeResult, } from './storage/index.js';
|
package/dist/index.js
CHANGED
|
@@ -27,6 +27,7 @@ export { readLessons, writeLessons, promoteLesson, matchLessons, lessonText, les
|
|
|
27
27
|
export { lessonsForDiff, lessonsSection } from './review-learning/team-lessons.js';
|
|
28
28
|
export { readRepoRules, rulesForDiff, rulesSection, splitRules } from './review-learning/repo-rules.js';
|
|
29
29
|
export { diffFromGit, GitDiffError } from './review/git-diff.js';
|
|
30
|
+
export { checkPrecisions, checkId, precisionOf, isMuted, MUTE_MIN_OUTCOMES, MUTE_BELOW } from './review/check-outcomes.js';
|
|
30
31
|
export { recordReviewOutcome, listResolvedFixes, removeResolvedFix, openFindingCount } from './review/agent-fixes.js';
|
|
31
32
|
export { computeEffectiveness, readAgentEvents, appendAgentEvent, findingKeys } from './review/effectiveness.js';
|
|
32
33
|
export { parseDiff, changedLinesByFile } from './utils/diff.js';
|
|
@@ -47,7 +48,9 @@ export { extractFacts, factsToPromptString } from './deep/fact-extractor.js';
|
|
|
47
48
|
export { openDatabase, isSQLiteAvailable, compactDatabase, getDatabaseSize, resetDatabase, insertScan, insertFindings, getRecentScans, getScoreTrendFromDB, getTopIssues, reinforcePattern, getStrongPatterns } from './storage/index.js';
|
|
48
49
|
// Local Project Memory (hybrid intelligence — SQLite-backed per-project learning)
|
|
49
50
|
export { checkLocalPatterns, persistAndReinforce, getProjectStats } from './storage/index.js';
|
|
50
|
-
export {
|
|
51
|
+
export { recordFixLessons, fixLessonSubject, fixLessonPrefix, FILES_TO_VALIDATE } from './storage/index.js';
|
|
52
|
+
export { recordInteractionEvidence, countInteractionEvidence, listLessons, listKnowledgeLessons, getApplicableLessons, transitionLesson, getRepositoryId, shareMemoryLesson, queueLocalLessonsForTeam } from './storage/index.js';
|
|
53
|
+
export { rankMemories, rankByMeaning, memoryText, SEMANTIC_FLOOR, RECALL_LIMIT } from './memory/recall.js';
|
|
51
54
|
export { loadTeamConfiguration, saveTeamConfiguration, initializeTeamSchema, getTeamModeStatus, doctorTeamConnection, syncTeamOutbox, searchTeamKnowledge, backfillTeamEmbeddings, TEAM_CONFIG_PATH, validateTeamDatabaseUrl, explainTeamConnectionError, } from './storage/index.js';
|
|
52
55
|
// Temporal Drift Engine (v5 — cross-session trend analysis + per-provenance EWMA)
|
|
53
56
|
export { generateTemporalDriftReport, formatDriftSummary } from './services/temporal-drift.js';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { spawnSync } from 'child_process';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import { describe, expect, it } from 'vitest';
|
|
6
|
+
const PACKAGES = path.resolve(__dirname, '../../..');
|
|
7
|
+
const scripts = fs.readdirSync(PACKAGES)
|
|
8
|
+
.filter(name => name.startsWith('brain-') && !name.includes('win'))
|
|
9
|
+
.map(name => path.join(PACKAGES, name, 'bin', 'rigour-brain'));
|
|
10
|
+
describe('rigour-brain placeholder', () => {
|
|
11
|
+
it.runIf(process.platform !== 'win32')('prints its message without running the command it names', () => {
|
|
12
|
+
const bin = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-'));
|
|
13
|
+
const marker = path.join(bin, 'ran');
|
|
14
|
+
fs.writeFileSync(path.join(bin, 'rigour'), `#!/bin/sh\necho "$@" > "${marker}"\n`, { mode: 0o755 });
|
|
15
|
+
try {
|
|
16
|
+
expect(scripts.length).toBeGreaterThan(0);
|
|
17
|
+
for (const script of scripts) {
|
|
18
|
+
const result = spawnSync('sh', [script, '--version'], { env: { PATH: `${bin}:/usr/bin:/bin` }, encoding: 'utf8' });
|
|
19
|
+
expect(result.status).toBe(1);
|
|
20
|
+
expect(result.stderr).toContain('rigour deep pull');
|
|
21
|
+
expect(fs.existsSync(marker)).toBe(false);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
finally {
|
|
25
|
+
fs.rmSync(bin, { recursive: true, force: true });
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import os from 'os';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { Readable } from 'stream';
|
|
5
|
+
import { afterAll, describe, expect, it, vi } from 'vitest';
|
|
6
|
+
/** Install into a scratch folder: the real one is ~/.rigour. */
|
|
7
|
+
const home = vi.hoisted(() => {
|
|
8
|
+
const p = require('path');
|
|
9
|
+
return p.join(require('os').tmpdir(), `rigour-engine-${process.pid}`);
|
|
10
|
+
});
|
|
11
|
+
vi.mock('../storage/db.js', async (importOriginal) => ({ ...(await importOriginal()), RIGOUR_DIR: home }));
|
|
12
|
+
const { installLlamaEngine, managedEngineDir, releaseAssetFor } = await import('./llama-engine.js');
|
|
13
|
+
afterAll(() => fs.rmSync(home, { recursive: true, force: true }));
|
|
14
|
+
describe('llama.cpp engine install', () => {
|
|
15
|
+
it('pins a checksum for every published platform', () => {
|
|
16
|
+
for (const key of ['darwin-arm64', 'darwin-x64', 'linux-x64', 'win32-x64']) {
|
|
17
|
+
expect(releaseAssetFor(key)?.sha256).toMatch(/^[0-9a-f]{64}$/);
|
|
18
|
+
}
|
|
19
|
+
expect(releaseAssetFor('linux-arm64')).toBeUndefined();
|
|
20
|
+
});
|
|
21
|
+
it.runIf(releaseAssetFor(`${os.platform()}-${os.arch()}`))('refuses an archive that does not match its pinned checksum, and installs nothing', async () => {
|
|
22
|
+
const tampered = async () => ({ statusCode: 200, headers: {}, body: Readable.from([Buffer.from('not the real llama.cpp archive')]) });
|
|
23
|
+
await expect(installLlamaEngine(undefined, tampered)).rejects.toThrow(/does not match its pinned checksum/);
|
|
24
|
+
expect(fs.existsSync(managedEngineDir())).toBe(false);
|
|
25
|
+
expect(fs.readdirSync(path.dirname(managedEngineDir())).filter(f => f.endsWith('.zip'))).toEqual([]);
|
|
26
|
+
});
|
|
27
|
+
});
|
|
@@ -4,7 +4,10 @@ export declare function llamaBinaryName(): string;
|
|
|
4
4
|
/** Install directory for the pinned release: ~/.rigour/bin/llama-<tag>. */
|
|
5
5
|
export declare function managedEngineDir(): string;
|
|
6
6
|
export declare function managedEnginePath(): string;
|
|
7
|
-
export declare function releaseAssetFor(platformKey: string):
|
|
7
|
+
export declare function releaseAssetFor(platformKey: string): {
|
|
8
|
+
file: string;
|
|
9
|
+
sha256: string;
|
|
10
|
+
} | undefined;
|
|
8
11
|
export interface ProbeResult {
|
|
9
12
|
ok: boolean;
|
|
10
13
|
version?: string;
|
|
@@ -16,12 +16,16 @@ export const LLAMA_RELEASE_TAG = 'b5604';
|
|
|
16
16
|
const LLAMA_RELEASE_BASE = `https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_RELEASE_TAG}`;
|
|
17
17
|
const PROBE_TIMEOUT_MS = 5_000;
|
|
18
18
|
const EXTRACT_TIMEOUT_MS = 120_000;
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* Platform → llama.cpp release asset and its SHA-256, checked against the digest GitHub publishes
|
|
21
|
+
* for the release. A download that does not match is refused: the engine runs code on the
|
|
22
|
+
* user's machine. linux-arm64 has no published build.
|
|
23
|
+
*/
|
|
20
24
|
const LLAMA_RELEASE_ASSETS = {
|
|
21
|
-
'darwin-arm64': `llama-${LLAMA_RELEASE_TAG}-bin-macos-arm64.zip`,
|
|
22
|
-
'darwin-x64': `llama-${LLAMA_RELEASE_TAG}-bin-macos-x64.zip`,
|
|
23
|
-
'linux-x64': `llama-${LLAMA_RELEASE_TAG}-bin-ubuntu-x64.zip`,
|
|
24
|
-
'win32-x64': `llama-${LLAMA_RELEASE_TAG}-bin-win-cpu-x64.zip`,
|
|
25
|
+
'darwin-arm64': { file: `llama-${LLAMA_RELEASE_TAG}-bin-macos-arm64.zip`, sha256: '673c5f8c4a7a2226ca6f3db5141d1ed78dcdf178d902c1486b77b3dda41e6a61' },
|
|
26
|
+
'darwin-x64': { file: `llama-${LLAMA_RELEASE_TAG}-bin-macos-x64.zip`, sha256: 'ab9ba59f3e355aebe6bc2c632b3489d4ab98d727f0ee03b9df6b4a6207452ad2' },
|
|
27
|
+
'linux-x64': { file: `llama-${LLAMA_RELEASE_TAG}-bin-ubuntu-x64.zip`, sha256: '3981c1e353399fbc35ea23316854b3c302877c00fb6b7fa4063a0623ad9e0309' },
|
|
28
|
+
'win32-x64': { file: `llama-${LLAMA_RELEASE_TAG}-bin-win-cpu-x64.zip`, sha256: 'df32d58de0b57c3c9de2e68e44b947efbac1b8a3e417536b6703cbd8da11eddd' },
|
|
25
29
|
};
|
|
26
30
|
export function llamaBinaryName() {
|
|
27
31
|
return os.platform() === 'win32' ? 'llama-cli.exe' : 'llama-cli';
|
|
@@ -84,13 +88,16 @@ export async function installLlamaEngine(onProgress, get) {
|
|
|
84
88
|
}
|
|
85
89
|
const installDir = managedEngineDir();
|
|
86
90
|
const workDir = `${installDir}.partial`;
|
|
87
|
-
const zipPath = path.join(path.dirname(installDir), asset);
|
|
91
|
+
const zipPath = path.join(path.dirname(installDir), asset.file);
|
|
88
92
|
await fs.ensureDir(path.dirname(installDir));
|
|
89
93
|
await fs.remove(workDir);
|
|
90
94
|
onProgress?.(`⬇ Downloading inference engine (llama.cpp ${LLAMA_RELEASE_TAG})...`);
|
|
91
95
|
try {
|
|
92
|
-
await downloadToFile(`${LLAMA_RELEASE_BASE}/${asset}`, zipPath, { get });
|
|
93
|
-
|
|
96
|
+
const { sha256 } = await downloadToFile(`${LLAMA_RELEASE_BASE}/${asset.file}`, zipPath, { get });
|
|
97
|
+
if (sha256 !== asset.sha256) {
|
|
98
|
+
throw new Error(`llama.cpp ${asset.file} does not match its pinned checksum (got ${sha256.slice(0, 12)}…, expected ${asset.sha256.slice(0, 12)}…); not installing it.`);
|
|
99
|
+
}
|
|
100
|
+
onProgress?.(' Verified checksum. Extracting...');
|
|
94
101
|
await extractZip(zipPath, workDir);
|
|
95
102
|
await installBinFolder(workDir, installDir);
|
|
96
103
|
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recall by meaning: the few memories that match what an agent is about to do,
|
|
3
|
+
* instead of every memory at once.
|
|
4
|
+
*
|
|
5
|
+
* Ranking uses the same local embedding model as the pattern index and team
|
|
6
|
+
* knowledge (all-MiniLM-L6-v2, loaded lazily so its native dependency never
|
|
7
|
+
* loads for anything else). When the model is unavailable it falls back to
|
|
8
|
+
* keyword overlap and says so in each result's `match`.
|
|
9
|
+
*/
|
|
10
|
+
export type MemoryScope = 'repo' | 'user';
|
|
11
|
+
export interface MemoryEntry {
|
|
12
|
+
scope: MemoryScope;
|
|
13
|
+
key: string;
|
|
14
|
+
value: string;
|
|
15
|
+
timestamp?: string;
|
|
16
|
+
}
|
|
17
|
+
export type RankedMemory = Ranked<MemoryEntry>;
|
|
18
|
+
/**
|
|
19
|
+
* The similarity a memory needs to be returned, for local memory and team knowledge alike. Measured
|
|
20
|
+
* with scripts/eval/memory-recall.mjs: at 0.25 recall and precision are both 81% and no unrelated
|
|
21
|
+
* query gets anything; at the previous 0.45, 62% of right memories were missed.
|
|
22
|
+
*/
|
|
23
|
+
export declare const SEMANTIC_FLOOR = 0.25;
|
|
24
|
+
/** Share of the query's words a memory must contain when ranking falls back to keywords. */
|
|
25
|
+
export declare const KEYWORD_FLOOR = 0.5;
|
|
26
|
+
export declare const RECALL_LIMIT = 3;
|
|
27
|
+
type Embed = (text: string) => Promise<number[]>;
|
|
28
|
+
export type Ranked<T> = T & {
|
|
29
|
+
score: number;
|
|
30
|
+
match: 'semantic' | 'keyword';
|
|
31
|
+
};
|
|
32
|
+
export declare function rankMemories(query: string, entries: MemoryEntry[], embed?: Embed): Promise<RankedMemory[]>;
|
|
33
|
+
/** The items whose text best matches the query by meaning, at most RECALL_LIMIT, each above the floor. */
|
|
34
|
+
export declare function rankByMeaning<T>(query: string, items: T[], textOf: (item: T) => string, embed?: Embed): Promise<Array<Ranked<T>>>;
|
|
35
|
+
/** What a memory is searched by: its key reads as a title (`db_access` → "db access"). */
|
|
36
|
+
export declare function memoryText(entry: Pick<MemoryEntry, 'key' | 'value'>): string;
|
|
37
|
+
export {};
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recall by meaning: the few memories that match what an agent is about to do,
|
|
3
|
+
* instead of every memory at once.
|
|
4
|
+
*
|
|
5
|
+
* Ranking uses the same local embedding model as the pattern index and team
|
|
6
|
+
* knowledge (all-MiniLM-L6-v2, loaded lazily so its native dependency never
|
|
7
|
+
* loads for anything else). When the model is unavailable it falls back to
|
|
8
|
+
* keyword overlap and says so in each result's `match`.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* The similarity a memory needs to be returned, for local memory and team knowledge alike. Measured
|
|
12
|
+
* with scripts/eval/memory-recall.mjs: at 0.25 recall and precision are both 81% and no unrelated
|
|
13
|
+
* query gets anything; at the previous 0.45, 62% of right memories were missed.
|
|
14
|
+
*/
|
|
15
|
+
export const SEMANTIC_FLOOR = 0.25;
|
|
16
|
+
/** Share of the query's words a memory must contain when ranking falls back to keywords. */
|
|
17
|
+
export const KEYWORD_FLOOR = 0.5;
|
|
18
|
+
export const RECALL_LIMIT = 3;
|
|
19
|
+
/** Memories considered per recall; more are skipped so a large store cannot stall an agent. */
|
|
20
|
+
const MAX_CANDIDATES = 300;
|
|
21
|
+
/**
|
|
22
|
+
* Embeddings by text, kept for the life of the process: the MCP server is long-lived and the same
|
|
23
|
+
* memories and lessons are ranked on every recall, so each text is embedded once, not per query.
|
|
24
|
+
*/
|
|
25
|
+
const EMBEDDING_CACHE_LIMIT = 2000;
|
|
26
|
+
const embeddingCache = new Map();
|
|
27
|
+
async function cachedEmbedding(embed, text) {
|
|
28
|
+
const hit = embeddingCache.get(text);
|
|
29
|
+
if (hit)
|
|
30
|
+
return hit;
|
|
31
|
+
const vector = await embed(text);
|
|
32
|
+
if (vector.length === 0)
|
|
33
|
+
return vector; // unavailable: do not remember a failure
|
|
34
|
+
if (embeddingCache.size >= EMBEDDING_CACHE_LIMIT)
|
|
35
|
+
embeddingCache.delete(embeddingCache.keys().next().value);
|
|
36
|
+
embeddingCache.set(text, vector);
|
|
37
|
+
return vector;
|
|
38
|
+
}
|
|
39
|
+
export async function rankMemories(query, entries, embed) {
|
|
40
|
+
return rankByMeaning(query, entries, memoryText, embed);
|
|
41
|
+
}
|
|
42
|
+
/** The items whose text best matches the query by meaning, at most RECALL_LIMIT, each above the floor. */
|
|
43
|
+
export async function rankByMeaning(query, items, textOf, embed) {
|
|
44
|
+
const candidates = items.slice(0, MAX_CANDIDATES);
|
|
45
|
+
if (candidates.length === 0)
|
|
46
|
+
return [];
|
|
47
|
+
const embedText = embed ?? (await import('../pattern-index/embeddings.js')).generateEmbedding;
|
|
48
|
+
const queryVector = await embedText(query);
|
|
49
|
+
const ranked = queryVector.length > 0
|
|
50
|
+
? await bySimilarity(queryVector, candidates, textOf, embedText)
|
|
51
|
+
: byKeywords(query, candidates, textOf);
|
|
52
|
+
return ranked.sort((a, b) => b.score - a.score).slice(0, RECALL_LIMIT);
|
|
53
|
+
}
|
|
54
|
+
async function bySimilarity(queryVector, items, textOf, embed) {
|
|
55
|
+
const ranked = [];
|
|
56
|
+
for (const item of items) {
|
|
57
|
+
const score = cosine(queryVector, await cachedEmbedding(embed, textOf(item)));
|
|
58
|
+
if (score >= SEMANTIC_FLOOR)
|
|
59
|
+
ranked.push({ ...item, score, match: 'semantic' });
|
|
60
|
+
}
|
|
61
|
+
return ranked;
|
|
62
|
+
}
|
|
63
|
+
function byKeywords(query, items, textOf) {
|
|
64
|
+
const wanted = words(query);
|
|
65
|
+
if (wanted.size === 0)
|
|
66
|
+
return [];
|
|
67
|
+
return items.flatMap(item => {
|
|
68
|
+
const have = words(textOf(item));
|
|
69
|
+
const score = [...wanted].filter(word => have.has(word)).length / wanted.size;
|
|
70
|
+
return score >= KEYWORD_FLOOR ? [{ ...item, score, match: 'keyword' }] : [];
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
/** What a memory is searched by: its key reads as a title (`db_access` → "db access"). */
|
|
74
|
+
export function memoryText(entry) {
|
|
75
|
+
return `${entry.key.replace(/[_-]+/g, ' ')}: ${entry.value}`;
|
|
76
|
+
}
|
|
77
|
+
const STOP_WORDS = new Set(['the', 'and', 'for', 'with', 'that', 'this', 'from', 'into', 'how', 'what', 'when', 'use', 'our', 'are', 'not']);
|
|
78
|
+
function words(text) {
|
|
79
|
+
return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter(word => word.length > 2 && !STOP_WORDS.has(word)));
|
|
80
|
+
}
|
|
81
|
+
function cosine(a, b) {
|
|
82
|
+
if (a.length === 0 || a.length !== b.length)
|
|
83
|
+
return 0;
|
|
84
|
+
let dot = 0;
|
|
85
|
+
let normA = 0;
|
|
86
|
+
let normB = 0;
|
|
87
|
+
for (let i = 0; i < a.length; i++) {
|
|
88
|
+
dot += a[i] * b[i];
|
|
89
|
+
normA += a[i] * a[i];
|
|
90
|
+
normB += b[i] * b[i];
|
|
91
|
+
}
|
|
92
|
+
return normA && normB ? dot / Math.sqrt(normA * normB) : 0;
|
|
93
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|