@pella-labs/pinakes 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -20
- package/dist/cli/audit.d.ts +5 -3
- package/dist/cli/audit.d.ts.map +1 -1
- package/dist/cli/audit.js +3 -24
- package/dist/cli/audit.js.map +1 -1
- package/dist/cli/contradiction-cli.d.ts +1 -0
- package/dist/cli/contradiction-cli.d.ts.map +1 -1
- package/dist/cli/contradiction-cli.js +12 -3
- package/dist/cli/contradiction-cli.js.map +1 -1
- package/dist/cli/contradiction.js +8 -8
- package/dist/cli/contradiction.js.map +1 -1
- package/dist/cli/export.d.ts +1 -1
- package/dist/cli/export.d.ts.map +1 -1
- package/dist/cli/export.js +7 -28
- package/dist/cli/export.js.map +1 -1
- package/dist/cli/import.d.ts +2 -2
- package/dist/cli/import.d.ts.map +1 -1
- package/dist/cli/import.js +5 -26
- package/dist/cli/import.js.map +1 -1
- package/dist/cli/index.js +17 -12
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/purge.d.ts +2 -2
- package/dist/cli/purge.d.ts.map +1 -1
- package/dist/cli/purge.js +2 -23
- package/dist/cli/purge.js.map +1 -1
- package/dist/cli/rebuild.d.ts +8 -6
- package/dist/cli/rebuild.d.ts.map +1 -1
- package/dist/cli/rebuild.js +9 -31
- package/dist/cli/rebuild.js.map +1 -1
- package/dist/cli/serve.d.ts +9 -7
- package/dist/cli/serve.d.ts.map +1 -1
- package/dist/cli/serve.js +71 -41
- package/dist/cli/serve.js.map +1 -1
- package/dist/cli/status.d.ts +7 -5
- package/dist/cli/status.d.ts.map +1 -1
- package/dist/cli/status.js +10 -29
- package/dist/cli/status.js.map +1 -1
- package/dist/db/client.d.ts +4 -4
- package/dist/db/client.js +3 -3
- package/dist/db/client.js.map +1 -1
- package/dist/db/migrations/0002_rename_kg_to_pinakes.sql +49 -0
- package/dist/db/migrations/{migrations/meta/0001_snapshot.json → meta/0002_snapshot.json} +44 -44
- package/dist/db/migrations/meta/_journal.json +7 -0
- package/dist/db/repository.js +5 -5
- package/dist/db/repository.js.map +1 -1
- package/dist/db/schema.d.ts +84 -84
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +49 -49
- package/dist/db/schema.js.map +1 -1
- package/dist/db/types.d.ts +2 -2
- package/dist/db/types.js +1 -1
- package/dist/gaps/detector.d.ts +5 -5
- package/dist/gaps/detector.js +15 -15
- package/dist/gaps/detector.js.map +1 -1
- package/dist/ingest/chokidar.js +1 -1
- package/dist/ingest/ingester.d.ts +4 -4
- package/dist/ingest/ingester.js +21 -21
- package/dist/ingest/ingester.js.map +1 -1
- package/dist/ingest/manifest.d.ts +5 -3
- package/dist/ingest/manifest.d.ts.map +1 -1
- package/dist/ingest/manifest.js +23 -14
- package/dist/ingest/manifest.js.map +1 -1
- package/dist/ingest/parse/chunk.js +1 -1
- package/dist/ingest/parse/markdown.d.ts +2 -2
- package/dist/ingest/queue.d.ts +1 -1
- package/dist/ingest/queue.js +1 -1
- package/dist/ingest/source.d.ts +2 -2
- package/dist/ingest/source.d.ts.map +1 -1
- package/dist/ingest/source.js +1 -1
- package/dist/llm/provider.js +3 -3
- package/dist/llm/provider.js.map +1 -1
- package/dist/mcp/envelope.d.ts +1 -1
- package/dist/mcp/tools/execute.d.ts +7 -7
- package/dist/mcp/tools/execute.d.ts.map +1 -1
- package/dist/mcp/tools/execute.js +27 -27
- package/dist/mcp/tools/execute.js.map +1 -1
- package/dist/mcp/tools/search.d.ts +8 -8
- package/dist/mcp/tools/search.d.ts.map +1 -1
- package/dist/mcp/tools/search.js +23 -22
- package/dist/mcp/tools/search.js.map +1 -1
- package/dist/observability/audit.d.ts +2 -2
- package/dist/observability/audit.js +3 -3
- package/dist/observability/audit.js.map +1 -1
- package/dist/observability/logger.js +4 -4
- package/dist/observability/logger.js.map +1 -1
- package/dist/observability/metrics.js +1 -1
- package/dist/observability/metrics.js.map +1 -1
- package/dist/paths.d.ts +61 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +106 -0
- package/dist/paths.js.map +1 -0
- package/dist/retrieval/embedder.d.ts +9 -9
- package/dist/retrieval/embedder.d.ts.map +1 -1
- package/dist/retrieval/embedder.js +17 -17
- package/dist/retrieval/embedder.js.map +1 -1
- package/dist/retrieval/fts.d.ts +3 -3
- package/dist/retrieval/fts.js +8 -8
- package/dist/retrieval/graph.d.ts +1 -1
- package/dist/retrieval/graph.js +8 -8
- package/dist/retrieval/graph.js.map +1 -1
- package/dist/retrieval/vec.d.ts +2 -2
- package/dist/retrieval/vec.js +3 -3
- package/dist/sandbox/bindings/install.d.ts +5 -5
- package/dist/sandbox/bindings/install.d.ts.map +1 -1
- package/dist/sandbox/bindings/install.js +3 -3
- package/dist/sandbox/bindings/install.js.map +1 -1
- package/dist/sandbox/bindings/pinakes.d.ts +29 -0
- package/dist/sandbox/bindings/pinakes.d.ts.map +1 -0
- package/dist/sandbox/bindings/pinakes.js +333 -0
- package/dist/sandbox/bindings/pinakes.js.map +1 -0
- package/dist/sandbox/bindings/write.d.ts +1 -1
- package/dist/sandbox/bindings/write.d.ts.map +1 -1
- package/dist/sandbox/bindings/write.js +9 -9
- package/dist/sandbox/bindings/write.js.map +1 -1
- package/dist/sandbox/executor.d.ts +3 -3
- package/dist/sandbox/executor.js +6 -6
- package/dist/sandbox/executor.js.map +1 -1
- package/dist/sandbox/helpers.d.ts +1 -1
- package/dist/sandbox/helpers.d.ts.map +1 -1
- package/dist/sandbox/helpers.js +2 -2
- package/dist/sandbox/vendored-codemode.d.ts +1 -1
- package/dist/sandbox/vendored-codemode.js +1 -1
- package/dist/server.js +15 -6
- package/dist/server.js.map +1 -1
- package/package.json +9 -3
- package/dist/db/migrations/migrations/0000_conscious_lockheed.sql +0 -115
- package/dist/db/migrations/migrations/0001_icy_dust.sql +0 -1
- package/dist/db/migrations/migrations/meta/0000_snapshot.json +0 -496
- package/dist/db/migrations/migrations/meta/_journal.json +0 -20
package/dist/db/schema.js
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
import { sql } from 'drizzle-orm';
|
|
2
2
|
import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlite-core';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* Pinakes Drizzle schema (presearch.md §2.3, CLAUDE.md §Database Rules).
|
|
5
5
|
*
|
|
6
|
-
* 8 logical tables + `
|
|
7
|
-
* virtual tables (`
|
|
6
|
+
* 8 logical tables + `pinakes_meta` for schema versioning. Two of the eight are
|
|
7
|
+
* virtual tables (`pinakes_chunks_fts`, `pinakes_chunks_vec`) that drizzle-kit can't
|
|
8
8
|
* model — they're created via raw SQL appended to the initial migration in
|
|
9
9
|
* src/db/migrations. The drizzle code below covers the 7 regular tables
|
|
10
|
-
* plus
|
|
10
|
+
* plus pinakes_meta.
|
|
11
11
|
*
|
|
12
12
|
* Invariants this schema MUST preserve:
|
|
13
13
|
*
|
|
14
|
-
* - `
|
|
14
|
+
* - `pinakes_nodes.id` is `sha1(scope + ':' + source_uri + ':' + section_path)`,
|
|
15
15
|
* set by the ingester (NOT auto-generated). Re-ingesting the same
|
|
16
16
|
* markdown produces identical ids — Phase 2's idempotent upsert relies
|
|
17
17
|
* on this. The DB never sees the hashing logic; it just stores the value
|
|
18
18
|
* and enforces uniqueness via the PK.
|
|
19
19
|
*
|
|
20
|
-
* - `
|
|
20
|
+
* - `pinakes_chunks.id` is `sha1(node_id + ':' + chunk_index)`, same idea.
|
|
21
21
|
*
|
|
22
22
|
* - `chunk_sha = sha1(chunk_text)` is the LOAD-BEARING field for the
|
|
23
23
|
* per-chunk skip-unchanged optimization (CLAUDE.md §Database Rules #3,
|
|
@@ -27,10 +27,10 @@ import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlit
|
|
|
27
27
|
* ~50ms = 3s of blocking work that competes with the active coding LLM
|
|
28
28
|
* for Ollama. Do not remove this column.
|
|
29
29
|
*
|
|
30
|
-
* - `last_accessed_at` on `
|
|
30
|
+
* - `last_accessed_at` on `pinakes_nodes` exists for the Phase 5 personal-KG
|
|
31
31
|
* LRU eviction (Loop 6.5 A2). Phase 2 just stamps it on insert/update.
|
|
32
32
|
*
|
|
33
|
-
* - `source_sha` on `
|
|
33
|
+
* - `source_sha` on `pinakes_nodes` is the file-level hash; staleness detection
|
|
34
34
|
* on the query path compares this against the current on-disk hash.
|
|
35
35
|
*
|
|
36
36
|
* - All FK relationships use `ON DELETE CASCADE` so deleting a node cleans
|
|
@@ -39,7 +39,7 @@ import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlit
|
|
|
39
39
|
* every connection in client.ts.
|
|
40
40
|
*/
|
|
41
41
|
// ----------------------------------------------------------------------------
|
|
42
|
-
//
|
|
42
|
+
// pinakes_meta — schema versioning + bookkeeping
|
|
43
43
|
// ----------------------------------------------------------------------------
|
|
44
44
|
/**
|
|
45
45
|
* Tiny key/value table holding `schema_version`, `last_full_rebuild`, and
|
|
@@ -50,12 +50,12 @@ import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlit
|
|
|
50
50
|
* (in the worst case for sqlite-vec breaking changes) drop + rebuild the
|
|
51
51
|
* vec virtual table from markdown.
|
|
52
52
|
*/
|
|
53
|
-
export const
|
|
53
|
+
export const pinakesMeta = sqliteTable('pinakes_meta', {
|
|
54
54
|
key: text('key').primaryKey(),
|
|
55
55
|
value: text('value'),
|
|
56
56
|
});
|
|
57
57
|
// ----------------------------------------------------------------------------
|
|
58
|
-
//
|
|
58
|
+
// pinakes_nodes — markdown sections + concept entities (Phase 2 only writes
|
|
59
59
|
// kind='section' rows; Phase 4 adds entities, Phase 6 adds gaps)
|
|
60
60
|
// ----------------------------------------------------------------------------
|
|
61
61
|
/**
|
|
@@ -71,11 +71,11 @@ export const kgMeta = sqliteTable('kg_meta', {
|
|
|
71
71
|
* Empty string for top-of-file content above any heading.
|
|
72
72
|
*
|
|
73
73
|
* `content` stores the full section markdown (heading + body). Chunks are
|
|
74
|
-
* derived from this and stored separately in
|
|
75
|
-
* `
|
|
74
|
+
* derived from this and stored separately in pinakes_chunks. We keep both because
|
|
75
|
+
* `pinakes_execute` callers may want the whole section (`pinakes.get(node_id)`) instead
|
|
76
76
|
* of paragraph-sized chunks.
|
|
77
77
|
*/
|
|
78
|
-
export const
|
|
78
|
+
export const pinakesNodes = sqliteTable('pinakes_nodes', {
|
|
79
79
|
/** sha1(scope + ':' + source_uri + ':' + section_path), set by ingester */
|
|
80
80
|
id: text('id').primaryKey(),
|
|
81
81
|
/** 'project' | 'personal' — enforced at app layer, not as a CHECK */
|
|
@@ -103,11 +103,11 @@ export const kgNodes = sqliteTable('kg_nodes', {
|
|
|
103
103
|
/** Unix epoch ms of last read access — Phase 5 LRU eviction (Loop 6.5 A2) */
|
|
104
104
|
lastAccessedAt: integer('last_accessed_at').notNull(),
|
|
105
105
|
}, (t) => [
|
|
106
|
-
index('
|
|
107
|
-
index('
|
|
106
|
+
index('idx_pinakes_nodes_scope_uri').on(t.scope, t.sourceUri),
|
|
107
|
+
index('idx_pinakes_nodes_last_accessed').on(t.lastAccessedAt),
|
|
108
108
|
]);
|
|
109
109
|
// ----------------------------------------------------------------------------
|
|
110
|
-
//
|
|
110
|
+
// pinakes_edges — wikilinks, citations, supersedes, etc. (Phase 4+ writes; Phase 2
|
|
111
111
|
// just creates the table for migration completeness)
|
|
112
112
|
// ----------------------------------------------------------------------------
|
|
113
113
|
/**
|
|
@@ -119,22 +119,22 @@ export const kgNodes = sqliteTable('kg_nodes', {
|
|
|
119
119
|
* markdown parsing. The table exists now so the migration is complete and
|
|
120
120
|
* Phase 4 doesn't have to add it as a separate migration.
|
|
121
121
|
*/
|
|
122
|
-
export const
|
|
122
|
+
export const pinakesEdges = sqliteTable('pinakes_edges', {
|
|
123
123
|
srcId: text('src_id')
|
|
124
124
|
.notNull()
|
|
125
|
-
.references(() =>
|
|
125
|
+
.references(() => pinakesNodes.id, { onDelete: 'cascade' }),
|
|
126
126
|
dstId: text('dst_id')
|
|
127
127
|
.notNull()
|
|
128
|
-
.references(() =>
|
|
128
|
+
.references(() => pinakesNodes.id, { onDelete: 'cascade' }),
|
|
129
129
|
/** 'wikilink' | 'cites' | 'supersedes' | 'contradicts' | 'mentions' | 'derived_from' */
|
|
130
130
|
edgeKind: text('edge_kind').notNull(),
|
|
131
131
|
}, (t) => [
|
|
132
132
|
primaryKey({ columns: [t.srcId, t.dstId, t.edgeKind] }),
|
|
133
|
-
index('
|
|
134
|
-
index('
|
|
133
|
+
index('idx_pinakes_edges_src').on(t.srcId),
|
|
134
|
+
index('idx_pinakes_edges_dst').on(t.dstId),
|
|
135
135
|
]);
|
|
136
136
|
// ----------------------------------------------------------------------------
|
|
137
|
-
//
|
|
137
|
+
// pinakes_chunks — paragraph-level splits of nodes
|
|
138
138
|
// ----------------------------------------------------------------------------
|
|
139
139
|
/**
|
|
140
140
|
* One row per ~500-token chunk derived from a node's `content`. The
|
|
@@ -143,8 +143,8 @@ export const kgEdges = sqliteTable('kg_edges', {
|
|
|
143
143
|
*
|
|
144
144
|
* The implicit SQLite `rowid` (auto-assigned for tables without
|
|
145
145
|
* INTEGER PRIMARY KEY) is what FTS5 and sqlite-vec join on:
|
|
146
|
-
* - `
|
|
147
|
-
* - `
|
|
146
|
+
* - `pinakes_chunks_fts` is `content='pinakes_chunks', content_rowid='rowid'`
|
|
147
|
+
* - `pinakes_chunks_vec.rowid` matches `pinakes_chunks.rowid`
|
|
148
148
|
*
|
|
149
149
|
* `chunk_sha = sha1(text)` is the per-chunk skip-unchanged key. On a
|
|
150
150
|
* file rewrite, the ingester compares each new chunk's chunk_sha against
|
|
@@ -153,13 +153,13 @@ export const kgEdges = sqliteTable('kg_edges', {
|
|
|
153
153
|
* re-embedded. This is the load-bearing optimization for Pharos's
|
|
154
154
|
* whole-file-rewrite-per-turn pattern (CLAUDE.md §Database Rules #3).
|
|
155
155
|
*/
|
|
156
|
-
export const
|
|
156
|
+
export const pinakesChunks = sqliteTable('pinakes_chunks', {
|
|
157
157
|
/** sha1(node_id + ':' + chunk_index) */
|
|
158
158
|
id: text('id').primaryKey(),
|
|
159
|
-
/** FK to
|
|
159
|
+
/** FK to pinakes_nodes; cascade-delete cleans up chunks when a node goes away */
|
|
160
160
|
nodeId: text('node_id')
|
|
161
161
|
.notNull()
|
|
162
|
-
.references(() =>
|
|
162
|
+
.references(() => pinakesNodes.id, { onDelete: 'cascade' }),
|
|
163
163
|
/** 0-based position within the node's chunk list */
|
|
164
164
|
chunkIndex: integer('chunk_index').notNull(),
|
|
165
165
|
/** The chunk's text content */
|
|
@@ -170,9 +170,9 @@ export const kgChunks = sqliteTable('kg_chunks', {
|
|
|
170
170
|
tokenCount: integer('token_count').notNull(),
|
|
171
171
|
/** Unix epoch ms of insert */
|
|
172
172
|
createdAt: integer('created_at').notNull(),
|
|
173
|
-
}, (t) => [index('
|
|
173
|
+
}, (t) => [index('idx_pinakes_chunks_node').on(t.nodeId)]);
|
|
174
174
|
// ----------------------------------------------------------------------------
|
|
175
|
-
//
|
|
175
|
+
// pinakes_log — append-only event log (Karpathy log.md materialized)
|
|
176
176
|
// ----------------------------------------------------------------------------
|
|
177
177
|
/**
|
|
178
178
|
* Append-only event stream. Phase 2 writes:
|
|
@@ -181,9 +181,9 @@ export const kgChunks = sqliteTable('kg_chunks', {
|
|
|
181
181
|
* - `'rebuild:start' | 'rebuild:done'` — full-rebuild markers
|
|
182
182
|
*
|
|
183
183
|
* `payload` is opaque JSON shaped per `kind`. The reader is the LLM via
|
|
184
|
-
* `
|
|
184
|
+
* `pinakes.log.recent(n, opts)` in Phase 4+; Phase 2 just appends.
|
|
185
185
|
*/
|
|
186
|
-
export const
|
|
186
|
+
export const pinakesLog = sqliteTable('pinakes_log', {
|
|
187
187
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
188
188
|
/** Unix epoch ms */
|
|
189
189
|
ts: integer('ts').notNull(),
|
|
@@ -195,9 +195,9 @@ export const kgLog = sqliteTable('kg_log', {
|
|
|
195
195
|
sourceUri: text('source_uri'),
|
|
196
196
|
/** Opaque JSON payload, shape per `kind` */
|
|
197
197
|
payload: text('payload'),
|
|
198
|
-
}, (t) => [index('
|
|
198
|
+
}, (t) => [index('idx_pinakes_log_ts').on(sql `${t.ts} DESC`)]);
|
|
199
199
|
// ----------------------------------------------------------------------------
|
|
200
|
-
//
|
|
200
|
+
// pinakes_gaps — detected concept gaps (Phase 6+ writes; Phase 2 creates table)
|
|
201
201
|
// ----------------------------------------------------------------------------
|
|
202
202
|
/**
|
|
203
203
|
* Concept gaps detected by the Phase 6 gap-detection sub-agent. Phase 2
|
|
@@ -207,7 +207,7 @@ export const kgLog = sqliteTable('kg_log', {
|
|
|
207
207
|
* multiple turns. `resolved_at` is set when a gap is closed (either by
|
|
208
208
|
* the LLM writing about the topic or by a manual dismissal).
|
|
209
209
|
*/
|
|
210
|
-
export const
|
|
210
|
+
export const pinakesGaps = sqliteTable('pinakes_gaps', {
|
|
211
211
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
212
212
|
scope: text('scope').notNull(),
|
|
213
213
|
topic: text('topic').notNull(),
|
|
@@ -216,7 +216,7 @@ export const kgGaps = sqliteTable('kg_gaps', {
|
|
|
216
216
|
resolvedAt: integer('resolved_at'),
|
|
217
217
|
});
|
|
218
218
|
// ----------------------------------------------------------------------------
|
|
219
|
-
//
|
|
219
|
+
// pinakes_audit — every tool call (Phase 5 wires writes; Phase 2 creates table)
|
|
220
220
|
// ----------------------------------------------------------------------------
|
|
221
221
|
/**
|
|
222
222
|
* Audit log of every MCP tool call. Phase 5 wires the dispatcher to write
|
|
@@ -224,12 +224,12 @@ export const kgGaps = sqliteTable('kg_gaps', {
|
|
|
224
224
|
* (CLAUDE.md §Security #7). Phase 2 creates the table.
|
|
225
225
|
*
|
|
226
226
|
* NB: per CLAUDE.md §Security #7, the JSONL mirror path differs by scope —
|
|
227
|
-
* project rows mirror to
|
|
228
|
-
*
|
|
229
|
-
* Personal-scope audit rows go to a separate
|
|
227
|
+
* project rows mirror to `~/.pinakes/projects/<mangled>/audit.jsonl`,
|
|
228
|
+
* personal/both rows mirror to `~/.pinakes/audit.jsonl`.
|
|
229
|
+
* Personal-scope audit rows go to a separate pinakes_audit table in the personal
|
|
230
230
|
* DB, never the project DB. This split is enforced at the app layer.
|
|
231
231
|
*/
|
|
232
|
-
export const
|
|
232
|
+
export const pinakesAudit = sqliteTable('pinakes_audit', {
|
|
233
233
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
234
234
|
ts: integer('ts').notNull(),
|
|
235
235
|
toolName: text('tool_name').notNull(),
|
|
@@ -242,18 +242,18 @@ export const kgAudit = sqliteTable('kg_audit', {
|
|
|
242
242
|
* The list of all schema-managed tables, useful for the schema test that
|
|
243
243
|
* verifies the migration creates every expected table.
|
|
244
244
|
*/
|
|
245
|
-
export const
|
|
246
|
-
'
|
|
247
|
-
'
|
|
248
|
-
'
|
|
249
|
-
'
|
|
250
|
-
'
|
|
251
|
-
'
|
|
252
|
-
'
|
|
245
|
+
export const PINAKES_TABLES = [
|
|
246
|
+
'pinakes_meta',
|
|
247
|
+
'pinakes_nodes',
|
|
248
|
+
'pinakes_edges',
|
|
249
|
+
'pinakes_chunks',
|
|
250
|
+
'pinakes_log',
|
|
251
|
+
'pinakes_gaps',
|
|
252
|
+
'pinakes_audit',
|
|
253
253
|
];
|
|
254
254
|
/**
|
|
255
255
|
* The two virtual tables created via raw SQL in the migration (drizzle-kit
|
|
256
256
|
* doesn't emit virtual table DDL). Schema test verifies they exist.
|
|
257
257
|
*/
|
|
258
|
-
export const
|
|
258
|
+
export const PINAKES_VIRTUAL_TABLES = ['pinakes_chunks_fts', 'pinakes_chunks_vec'];
|
|
259
259
|
//# sourceMappingURL=schema.js.map
|
package/dist/db/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,+EAA+E;AAC/E,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,+EAA+E;AAC/E,iDAAiD;AACjD,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,WAAW,CAAC,cAAc,EAAE;IACrD,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE;IAC7B,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;CACrB,CAAC,CAAC;AAEH,+EAA+E;AAC/E,4EAA4E;AAC5E,iEAAiE;AACjE,+EAA+E;AAE/E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CACrC,eAAe,EACf;IACE,2EAA2E;IAC3E,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,qEAAqE;IACrE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IAC9B,mFAAmF;IACnF,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IACvC,8EAA8E;IAC9E,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE;IAC3C,gDAAgD;IAChD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IAC/C,mEAAmE;IACnE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;IACpB,6CAA6C;IAC7C,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE;IAClC,+DAA+D;IAC/D,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IACvC,2DAA2D;IAC3D,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE;IAC5C,qGAAqG;IACrG,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC;IAC7D,oCAAoC;IACpC,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC1C,mCAAmC;IACnC,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC1C,6EAA6E;IAC7E,cAAc,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,OAAO,EAAE;CACtD,EACD,CAAC,CAAC,EAAE,EAAE,CAAC;IACL,KAAK,CAAC,6BAA6B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,SAAS,CAAC;IAC7D,KAAK,CAAC,iCAAiC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;CAC9D,CACF,CAAC;AAEF,+EAA+E;AAC/E,mFAAmF;AACnF,qDAAqD;AACrD,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CACrC,eAAe,EACf;IACE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC;SAClB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC7D,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC;SAClB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC7D,wFAAwF;IACxF,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;CACtC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC;IACL,UAAU,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvD,KAAK,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1C,KAAK,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;CAC3C,CACF,CAAC;AAEF,+EAA+E;AAC/E,mDAAmD;AACnD,+EAA+E;AAE/E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,WAAW,CACtC,gBAAgB,EAChB;IACE,wCAAwC;IACxC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,iFAAiF;IACjF,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC7D,oDAAoD;IACpD,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE;IAC5C,+BAA+B;IAC/B,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAC5B,6DAA6D;IAC7D,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;IACrC,8CAA8C;IAC9C,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE;IAC5C,8BAA8B;IAC9B,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;CAC3C,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CACvD,CAAC;AAEF,+EAA+E;AAC/E,qEAAqE;AACrE,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,WAAW,CACnC,aAAa,EACb;IACE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IACrD,oBAAoB;IACpB,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE;IAC3B,6BAA6B;IAC7B,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IAC9B,qCAAqC;IACrC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAC5B,6DAA6D;IAC7D,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,4CAA4C;IAC5C,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;CACzB,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,EAAE,CAAC,GAAG,CAAA,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAC3D,CAAC;AAEF,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,WAAW,CAAC,cAAc,EAAE;IACrD,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IACrD,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IAC9B,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IAC9B,WAAW,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;IAC/C,aAAa,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7D,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC;CACnC,CAAC,CAAC;AAEH,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC,eAAe,EAAE;IACvD,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IACrD,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE;IAC3B,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;IACrC,cAAc,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,OAAO,EAAE;IACjD,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,cAAc,EAAE,OAAO,CAAC,iBAAiB,CAAC;IAC1C,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;CACrB,CAAC,CAAC;AAeH;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,cAAc;IACd,eAAe;IACf,eAAe;IACf,gBAAgB;IAChB,aAAa;IACb,cAAc;IACd,eAAe;CACP,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,oBAAoB,EAAE,oBAAoB,CAAU,CAAC"}
|
package/dist/db/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Shared types for
|
|
2
|
+
* Shared types for Pinakes query results.
|
|
3
3
|
*
|
|
4
4
|
* Phase 2 introduces this file to decouple the tool handlers from any
|
|
5
5
|
* single store implementation. Phase 1's `MemoryStore` defined `Chunk`
|
|
@@ -12,7 +12,7 @@ export type Scope = 'project' | 'personal' | 'both';
|
|
|
12
12
|
* A retrieval-shaped chunk. Mirrors what Phase 1's `MemoryStore` returned
|
|
13
13
|
* so the existing 13 spike tests stay green across the swap. Field shape
|
|
14
14
|
* is locked — adding fields is fine; renaming/removing breaks the
|
|
15
|
-
* sandbox host bindings (`
|
|
15
|
+
* sandbox host bindings (`pinakes.search` returns `{id, text, source_uri}`)
|
|
16
16
|
* and the spike test asserting that shape.
|
|
17
17
|
*/
|
|
18
18
|
export interface Chunk {
|
package/dist/db/types.js
CHANGED
package/dist/gaps/detector.d.ts
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import type { Database as BetterSqliteDatabase } from 'better-sqlite3';
|
|
2
2
|
/**
|
|
3
|
-
* Gap detector for
|
|
3
|
+
* Gap detector for Pinakes Phase 6.
|
|
4
4
|
*
|
|
5
5
|
* After an ingest transaction commits, scans the new node's content for
|
|
6
6
|
* concept mentions. A "concept" is a term referenced via bold (`**term**`),
|
|
7
7
|
* wikilinks (`[[term]]`), or backtick-quoted identifiers that appears ≥3
|
|
8
|
-
* times across the entire KG but has no dedicated `
|
|
8
|
+
* times across the entire KG but has no dedicated `pinakes_nodes` row (i.e., no
|
|
9
9
|
* node whose title matches the concept).
|
|
10
10
|
*
|
|
11
|
-
* Upserts into `
|
|
11
|
+
* Upserts into `pinakes_gaps` with `topic`, `first_seen_at`, `mentions_count`.
|
|
12
12
|
* When a node is later created with a matching title, `resolved_at` is set.
|
|
13
13
|
*
|
|
14
14
|
* This is a read-only detection surface — the LLM fills gaps by calling
|
|
15
|
-
* `
|
|
15
|
+
* `pinakes.project.write()` to create wiki pages, and re-indexing resolves
|
|
16
16
|
* the gap automatically.
|
|
17
17
|
*/
|
|
18
18
|
/**
|
|
@@ -32,7 +32,7 @@ export declare function extractConcepts(content: string): Set<string>;
|
|
|
32
32
|
* For each concept extracted from the ingested content:
|
|
33
33
|
* 1. Count how many chunks across the KG contain the concept (case-insensitive)
|
|
34
34
|
* 2. Check if a node with a matching title already exists
|
|
35
|
-
* 3. If ≥3 mentions and no dedicated node → upsert into `
|
|
35
|
+
* 3. If ≥3 mentions and no dedicated node → upsert into `pinakes_gaps`
|
|
36
36
|
*
|
|
37
37
|
* Also resolves any existing gaps that now have a dedicated node.
|
|
38
38
|
*
|
package/dist/gaps/detector.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { logger } from '../observability/logger.js';
|
|
2
2
|
/**
|
|
3
|
-
* Gap detector for
|
|
3
|
+
* Gap detector for Pinakes Phase 6.
|
|
4
4
|
*
|
|
5
5
|
* After an ingest transaction commits, scans the new node's content for
|
|
6
6
|
* concept mentions. A "concept" is a term referenced via bold (`**term**`),
|
|
7
7
|
* wikilinks (`[[term]]`), or backtick-quoted identifiers that appears ≥3
|
|
8
|
-
* times across the entire KG but has no dedicated `
|
|
8
|
+
* times across the entire KG but has no dedicated `pinakes_nodes` row (i.e., no
|
|
9
9
|
* node whose title matches the concept).
|
|
10
10
|
*
|
|
11
|
-
* Upserts into `
|
|
11
|
+
* Upserts into `pinakes_gaps` with `topic`, `first_seen_at`, `mentions_count`.
|
|
12
12
|
* When a node is later created with a matching title, `resolved_at` is set.
|
|
13
13
|
*
|
|
14
14
|
* This is a read-only detection surface — the LLM fills gaps by calling
|
|
15
|
-
* `
|
|
15
|
+
* `pinakes.project.write()` to create wiki pages, and re-indexing resolves
|
|
16
16
|
* the gap automatically.
|
|
17
17
|
*/
|
|
18
18
|
// ----------------------------------------------------------------------------
|
|
@@ -65,7 +65,7 @@ export function extractConcepts(content) {
|
|
|
65
65
|
* For each concept extracted from the ingested content:
|
|
66
66
|
* 1. Count how many chunks across the KG contain the concept (case-insensitive)
|
|
67
67
|
* 2. Check if a node with a matching title already exists
|
|
68
|
-
* 3. If ≥3 mentions and no dedicated node → upsert into `
|
|
68
|
+
* 3. If ≥3 mentions and no dedicated node → upsert into `pinakes_gaps`
|
|
69
69
|
*
|
|
70
70
|
* Also resolves any existing gaps that now have a dedicated node.
|
|
71
71
|
*
|
|
@@ -84,16 +84,16 @@ export function detectGaps(writer, scope, content, nodesTitles) {
|
|
|
84
84
|
if (concepts.size === 0) {
|
|
85
85
|
return { gaps_created: gapsCreated, gaps_resolved: gapsResolved };
|
|
86
86
|
}
|
|
87
|
-
const countChunkMentions = writer.prepare(`SELECT count(*) AS c FROM
|
|
88
|
-
JOIN
|
|
87
|
+
const countChunkMentions = writer.prepare(`SELECT count(*) AS c FROM pinakes_chunks ch
|
|
88
|
+
JOIN pinakes_nodes n ON ch.node_id = n.id
|
|
89
89
|
WHERE n.scope = ? AND ch.text LIKE '%' || ? || '%' COLLATE NOCASE`);
|
|
90
|
-
const findDedicatedNode = writer.prepare(`SELECT id FROM
|
|
91
|
-
//
|
|
90
|
+
const findDedicatedNode = writer.prepare(`SELECT id FROM pinakes_nodes WHERE scope = ? AND LOWER(title) = ? LIMIT 1`);
|
|
91
|
+
// pinakes_gaps doesn't have a unique constraint on (scope, topic).
|
|
92
92
|
// Check if the gap already exists and update or insert accordingly.
|
|
93
|
-
const findGap = writer.prepare(`SELECT id, resolved_at FROM
|
|
94
|
-
const insertGap = writer.prepare(`INSERT INTO
|
|
93
|
+
const findGap = writer.prepare(`SELECT id, resolved_at FROM pinakes_gaps WHERE scope = ? AND topic = ? LIMIT 1`);
|
|
94
|
+
const insertGap = writer.prepare(`INSERT INTO pinakes_gaps (scope, topic, first_seen_at, mentions_count)
|
|
95
95
|
VALUES (?, ?, ?, ?)`);
|
|
96
|
-
const updateGapCount = writer.prepare(`UPDATE
|
|
96
|
+
const updateGapCount = writer.prepare(`UPDATE pinakes_gaps SET mentions_count = ?, resolved_at = NULL WHERE id = ?`);
|
|
97
97
|
const now = Date.now();
|
|
98
98
|
for (const concept of concepts) {
|
|
99
99
|
// Count mentions across the KG
|
|
@@ -130,7 +130,7 @@ export function resolveGaps(writer, scope, nodesTitles) {
|
|
|
130
130
|
return 0;
|
|
131
131
|
const now = Date.now();
|
|
132
132
|
let resolved = 0;
|
|
133
|
-
const resolveStmt = writer.prepare(`UPDATE
|
|
133
|
+
const resolveStmt = writer.prepare(`UPDATE pinakes_gaps SET resolved_at = ?
|
|
134
134
|
WHERE scope = ? AND LOWER(topic) = ? AND resolved_at IS NULL`);
|
|
135
135
|
for (const title of nodesTitles) {
|
|
136
136
|
if (!title)
|
|
@@ -148,12 +148,12 @@ export function queryGaps(reader, scope, opts) {
|
|
|
148
148
|
if (opts?.resolved) {
|
|
149
149
|
return reader
|
|
150
150
|
.prepare(`SELECT id, topic, first_seen_at, mentions_count, resolved_at
|
|
151
|
-
FROM
|
|
151
|
+
FROM pinakes_gaps WHERE scope = ? ORDER BY mentions_count DESC`)
|
|
152
152
|
.all(scope);
|
|
153
153
|
}
|
|
154
154
|
return reader
|
|
155
155
|
.prepare(`SELECT id, topic, first_seen_at, mentions_count, resolved_at
|
|
156
|
-
FROM
|
|
156
|
+
FROM pinakes_gaps WHERE scope = ? AND resolved_at IS NULL
|
|
157
157
|
ORDER BY mentions_count DESC`)
|
|
158
158
|
.all(scope);
|
|
159
159
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"detector.js","sourceRoot":"","sources":["../../src/gaps/detector.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEpD;;;;;;;;;;;;;;;GAeG;AAEH,+EAA+E;AAC/E,qBAAqB;AACrB,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,6BAA6B;IAC7B,MAAM,MAAM,GAAG,8BAA8B,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,CAAC;IAED,0CAA0C;IAC1C,MAAM,UAAU,GAAG,iCAAiC,CAAC;IACrD,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,CAAC;IAED,oDAAoD;IACpD,MAAM,UAAU,GAAG,0BAA0B,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACnD,gFAAgF;QAChF,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,6BAA6B;AAC7B,+EAA+E;AAE/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,UAAU,CACxB,MAA4B,EAC5B,KAAa,EACb,OAAe,EACf,WAAqB;IAErB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,+EAA+E;IAC/E,YAAY,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;IAEvD,iEAAiE;IACjE,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;IACpE,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CACvC;;wEAEoE,CACrE,CAAC;IAEF,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,CACtC,
|
|
1
|
+
{"version":3,"file":"detector.js","sourceRoot":"","sources":["../../src/gaps/detector.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEpD;;;;;;;;;;;;;;;GAeG;AAEH,+EAA+E;AAC/E,qBAAqB;AACrB,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,6BAA6B;IAC7B,MAAM,MAAM,GAAG,8BAA8B,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,CAAC;IAED,0CAA0C;IAC1C,MAAM,UAAU,GAAG,iCAAiC,CAAC;IACrD,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,CAAC;IAED,oDAAoD;IACpD,MAAM,UAAU,GAAG,0BAA0B,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACnD,gFAAgF;QAChF,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,6BAA6B;AAC7B,+EAA+E;AAE/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,UAAU,CACxB,MAA4B,EAC5B,KAAa,EACb,OAAe,EACf,WAAqB;IAErB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,+EAA+E;IAC/E,YAAY,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;IAEvD,iEAAiE;IACjE,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;IACpE,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CACvC;;wEAEoE,CACrE,CAAC;IAEF,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,CACtC,2EAA2E,CAC5E,CAAC;IAEF,mEAAmE;IACnE,oEAAoE;IACpE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAC5B,gFAAgF,CACjF,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAC9B;yBACqB,CACtB,CAAC;IAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CACnC,6EAA6E,CAC9E,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,+BAA+B;QAC/B,MAAM,GAAG,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,KAAK,GAAG,CAAC;YAAE,SAAS;QAExB,6BAA6B;QAC7B,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACxD,IAAI,SAAS;YAAE,SAAS;QAExB,iBAAiB;QACjB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,sDAAsD;YACtD,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;YAC1C,WAAW,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACxC,MAAM,CAAC,IAAI,CACT,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,eAAe,EAAE,QAAQ,CAAC,IAAI,EAAE,EACpE,wBAAwB,CACzB,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;AACpE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,MAA4B,EAC5B,KAAa,EACb,WAAqB;IAErB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAChC;mEAC+D,CAChE,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;QAChC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC9D,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC;IAC3B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAcD;;;GAGG;AACH,MAAM,UAAU,SAAS,CACvB,MAA4B,EAC5B,KAAa,EACb,IAA6B;IAE7B,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;QACnB,OAAO,MAAM;aACV,OAAO,CACN;0EACkE,CACnE;aACA,GAAG,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC;IAED,OAAO,MAAM;SACV,OAAO,CACN;;sCAEgC,CACjC;SACA,GAAG,CAAC,KAAK,CAAC,CAAC;AAChB,CAAC"}
|
package/dist/ingest/chokidar.js
CHANGED
|
@@ -2,7 +2,7 @@ import { resolve } from 'node:path';
|
|
|
2
2
|
import chokidar from 'chokidar';
|
|
3
3
|
import { logger } from '../observability/logger.js';
|
|
4
4
|
/**
|
|
5
|
-
* `ChokidarWatcher` — file-system implementation of `IngestSource` for
|
|
5
|
+
* `ChokidarWatcher` — file-system implementation of `IngestSource` for Pinakes Phase 2.
|
|
6
6
|
*
|
|
7
7
|
* Watches a wiki directory recursively for `*.md` changes, with the two
|
|
8
8
|
* critical adaptations for Pharos's wiki-updater (CLAUDE.md §Database Rules #4,
|
|
@@ -3,12 +3,12 @@ import type { Embedder } from '../retrieval/embedder.js';
|
|
|
3
3
|
import { type Manifest, fileSha, manifestPathFor } from './manifest.js';
|
|
4
4
|
import { type Chunk as ChunkText } from './parse/chunk.js';
|
|
5
5
|
/**
|
|
6
|
-
* IngesterService — the load-bearing single-flight writer for
|
|
6
|
+
* IngesterService — the load-bearing single-flight writer for Pinakes Phase 2.
|
|
7
7
|
*
|
|
8
8
|
* **What it does**: given an absolute path to a markdown file, parse it into
|
|
9
9
|
* sections, chunk each section, embed only the chunks whose sha changed, and
|
|
10
10
|
* upsert the result into SQLite under one transaction. Update the consistency
|
|
11
|
-
* manifest, append a row to `
|
|
11
|
+
* manifest, append a row to `pinakes_log`. Errors trigger a clean rollback and an
|
|
12
12
|
* error log entry, but never throw past the caller's await — they're caught,
|
|
13
13
|
* logged, and re-thrown so the rebuild CLI can decide whether to abort or
|
|
14
14
|
* continue.
|
|
@@ -21,7 +21,7 @@ import { type Chunk as ChunkText } from './parse/chunk.js';
|
|
|
21
21
|
* single-flight pattern; CLAUDE.md §Architecture #3 / §Database Rules #5.
|
|
22
22
|
*
|
|
23
23
|
* - **Per-chunk skip-unchanged**: on a re-ingest of an existing file, we
|
|
24
|
-
* load the existing `
|
|
24
|
+
* load the existing `pinakes_chunks` rows for that file's nodes, build a
|
|
25
25
|
* `Set<chunk_sha>` of already-embedded chunks, and only call `embedder.embed()`
|
|
26
26
|
* for chunks whose sha is NOT in the set. This is THE optimization that
|
|
27
27
|
* makes Pharos's whole-file-rewrite-per-turn pattern viable — without it,
|
|
@@ -108,7 +108,7 @@ export declare class IngesterService {
|
|
|
108
108
|
private appendLog;
|
|
109
109
|
}
|
|
110
110
|
/**
|
|
111
|
-
* Reset the in-memory manifest to empty. Used by `
|
|
111
|
+
* Reset the in-memory manifest to empty. Used by `pinakes rebuild` when the
|
|
112
112
|
* caller passes `--clean` to force a full re-ingest from scratch.
|
|
113
113
|
*/
|
|
114
114
|
export declare function freshManifest(): Manifest;
|
package/dist/ingest/ingester.js
CHANGED
|
@@ -81,7 +81,7 @@ export class IngesterService {
|
|
|
81
81
|
const abs = resolve(absPath);
|
|
82
82
|
const sourceUri = toStoredUri(abs, this.wikiRoot, this.scope);
|
|
83
83
|
this.bundle.writer
|
|
84
|
-
.prepare('DELETE FROM
|
|
84
|
+
.prepare('DELETE FROM pinakes_nodes WHERE scope = ? AND source_uri = ?')
|
|
85
85
|
.run(this.scope, sourceUri);
|
|
86
86
|
this.manifest = removeManifestEntry(this.manifest, abs, this.wikiRoot, this.scope);
|
|
87
87
|
writeManifest(this.manifestPath, this.manifest);
|
|
@@ -170,37 +170,37 @@ export class IngesterService {
|
|
|
170
170
|
const now = Date.now();
|
|
171
171
|
const totalChunks = planned.reduce((acc, n) => acc + n.chunks.length, 0);
|
|
172
172
|
this.runInTransaction((w) => {
|
|
173
|
-
// Delete old vec rows BEFORE deleting
|
|
174
|
-
// the rowids first because the FK cascade will drop the
|
|
173
|
+
// Delete old vec rows BEFORE deleting pinakes_chunks rows. We need to know
|
|
174
|
+
// the rowids first because the FK cascade will drop the pinakes_chunks rows
|
|
175
175
|
// (and sqlite-vec doesn't have FK awareness).
|
|
176
176
|
const oldRowids = w
|
|
177
177
|
.prepare(`SELECT c.rowid AS rowid
|
|
178
|
-
FROM
|
|
178
|
+
FROM pinakes_chunks c JOIN pinakes_nodes n ON c.node_id = n.id
|
|
179
179
|
WHERE n.scope = ? AND n.source_uri = ?`)
|
|
180
180
|
.all(this.scope, sourceUri);
|
|
181
181
|
if (oldRowids.length > 0) {
|
|
182
182
|
const placeholders = oldRowids.map(() => '?').join(',');
|
|
183
|
-
w.prepare(`DELETE FROM
|
|
183
|
+
w.prepare(`DELETE FROM pinakes_chunks_vec WHERE rowid IN (${placeholders})`).run(...oldRowids.map((r) => r.rowid));
|
|
184
184
|
}
|
|
185
185
|
// Delete old nodes for this file (cascades to chunks → FTS5 trigger fires).
|
|
186
186
|
// Also delete outbound edges from this file's nodes (edges don't cascade
|
|
187
|
-
// from
|
|
187
|
+
// from pinakes_nodes FK — they reference src_id/dst_id without ON DELETE CASCADE
|
|
188
188
|
// on dst_id, so we delete src edges explicitly).
|
|
189
189
|
const oldNodeIds = w
|
|
190
|
-
.prepare(`SELECT id FROM
|
|
190
|
+
.prepare(`SELECT id FROM pinakes_nodes WHERE scope = ? AND source_uri = ?`)
|
|
191
191
|
.all(this.scope, sourceUri)
|
|
192
192
|
.map((r) => r.id);
|
|
193
193
|
if (oldNodeIds.length > 0) {
|
|
194
194
|
const ph = oldNodeIds.map(() => '?').join(',');
|
|
195
|
-
w.prepare(`DELETE FROM
|
|
195
|
+
w.prepare(`DELETE FROM pinakes_edges WHERE src_id IN (${ph})`).run(...oldNodeIds);
|
|
196
196
|
}
|
|
197
|
-
w.prepare('DELETE FROM
|
|
197
|
+
w.prepare('DELETE FROM pinakes_nodes WHERE scope = ? AND source_uri = ?').run(this.scope, sourceUri);
|
|
198
198
|
// Insert new nodes
|
|
199
|
-
const insertNode = w.prepare(`INSERT INTO
|
|
199
|
+
const insertNode = w.prepare(`INSERT INTO pinakes_nodes (id, scope, source_uri, section_path, kind, title, content, source_sha, token_count, created_at, updated_at, last_accessed_at, confidence)
|
|
200
200
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
201
|
-
const insertChunk = w.prepare(`INSERT INTO
|
|
201
|
+
const insertChunk = w.prepare(`INSERT INTO pinakes_chunks (id, node_id, chunk_index, text, chunk_sha, token_count, created_at)
|
|
202
202
|
VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
203
|
-
const insertVec = w.prepare('INSERT INTO
|
|
203
|
+
const insertVec = w.prepare('INSERT INTO pinakes_chunks_vec(rowid, embedding) VALUES (?, ?)');
|
|
204
204
|
for (const node of planned) {
|
|
205
205
|
const sectionTokenCount = countTokens(node.section.content);
|
|
206
206
|
insertNode.run(node.nodeId, this.scope, sourceUri, node.section.section_path, node.section.kind, node.section.title || null, node.section.content, sourceSha, sectionTokenCount, now, now, now, confidence);
|
|
@@ -226,7 +226,7 @@ export class IngesterService {
|
|
|
226
226
|
// Scan each node's content for [[wikilinks]], resolve targets to
|
|
227
227
|
// existing node IDs, and insert edges. Unresolved links are silently
|
|
228
228
|
// skipped (gap detector already surfaces those).
|
|
229
|
-
const insertEdge = w.prepare(`INSERT OR IGNORE INTO
|
|
229
|
+
const insertEdge = w.prepare(`INSERT OR IGNORE INTO pinakes_edges (src_id, dst_id, edge_kind) VALUES (?, ?, 'wikilink')`);
|
|
230
230
|
for (const node of planned) {
|
|
231
231
|
const links = extractWikilinks(node.section.content);
|
|
232
232
|
for (const linkTarget of links) {
|
|
@@ -237,7 +237,7 @@ export class IngesterService {
|
|
|
237
237
|
}
|
|
238
238
|
}
|
|
239
239
|
});
|
|
240
|
-
// Step 7: append
|
|
240
|
+
// Step 7: append pinakes_log row
|
|
241
241
|
this.appendLog('ingest:done', absPath, {
|
|
242
242
|
nodes: planned.length,
|
|
243
243
|
chunks_added: totalChunks - chunksSkipped,
|
|
@@ -280,9 +280,9 @@ export class IngesterService {
|
|
|
280
280
|
const out = new Map();
|
|
281
281
|
const rows = this.bundle.writer
|
|
282
282
|
.prepare(`SELECT c.chunk_sha AS chunk_sha, vec.embedding AS embedding
|
|
283
|
-
FROM
|
|
284
|
-
JOIN
|
|
285
|
-
LEFT JOIN
|
|
283
|
+
FROM pinakes_chunks c
|
|
284
|
+
JOIN pinakes_nodes n ON c.node_id = n.id
|
|
285
|
+
LEFT JOIN pinakes_chunks_vec vec ON vec.rowid = c.rowid
|
|
286
286
|
WHERE n.scope = ? AND n.source_uri = ?`)
|
|
287
287
|
.all(this.scope, sourceUri);
|
|
288
288
|
for (const row of rows) {
|
|
@@ -304,12 +304,12 @@ export class IngesterService {
|
|
|
304
304
|
appendLog(kind, absPath, payload) {
|
|
305
305
|
try {
|
|
306
306
|
this.bundle.writer
|
|
307
|
-
.prepare(`INSERT INTO
|
|
307
|
+
.prepare(`INSERT INTO pinakes_log (ts, scope, kind, source_uri, payload)
|
|
308
308
|
VALUES (?, ?, ?, ?, ?)`)
|
|
309
309
|
.run(Date.now(), this.scope, kind, toStoredUri(absPath, this.wikiRoot, this.scope), JSON.stringify(payload));
|
|
310
310
|
}
|
|
311
311
|
catch (err) {
|
|
312
|
-
logger.warn({ err, kind, absPath }, 'failed to append
|
|
312
|
+
logger.warn({ err, kind, absPath }, 'failed to append pinakes_log row');
|
|
313
313
|
}
|
|
314
314
|
}
|
|
315
315
|
}
|
|
@@ -328,7 +328,7 @@ function sha1(input) {
|
|
|
328
328
|
// Cold-start helper for the rebuild CLI
|
|
329
329
|
// ----------------------------------------------------------------------------
|
|
330
330
|
/**
|
|
331
|
-
* Reset the in-memory manifest to empty. Used by `
|
|
331
|
+
* Reset the in-memory manifest to empty. Used by `pinakes rebuild` when the
|
|
332
332
|
* caller passes `--clean` to force a full re-ingest from scratch.
|
|
333
333
|
*/
|
|
334
334
|
export function freshManifest() {
|
|
@@ -364,7 +364,7 @@ function resolveWikilinkTarget(w, scope, target) {
|
|
|
364
364
|
// We match the last path component without .md extension.
|
|
365
365
|
const withMd = target.endsWith('.md') ? target : `${target}.md`;
|
|
366
366
|
const row = w
|
|
367
|
-
.prepare(`SELECT id FROM
|
|
367
|
+
.prepare(`SELECT id FROM pinakes_nodes
|
|
368
368
|
WHERE scope = ? AND (
|
|
369
369
|
source_uri = ? OR
|
|
370
370
|
source_uri LIKE ? OR
|