@gmickel/gno 1.9.0 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/skill/SKILL.md +28 -16
- package/assets/skill/cli-reference.md +30 -0
- package/assets/skill/examples.md +20 -0
- package/assets/skill/mcp-reference.md +7 -1
- package/package.json +1 -1
- package/src/cli/commands/graph.ts +89 -2
- package/src/cli/commands/links.ts +237 -54
- package/src/cli/commands/query.ts +212 -0
- package/src/cli/commands/ref-parser.ts +8 -103
- package/src/cli/options.ts +4 -0
- package/src/cli/program.ts +176 -9
- package/src/config/content-types.ts +14 -0
- package/src/core/file-lock.ts +38 -3
- package/src/core/graph-query.ts +137 -0
- package/src/core/graph-resolver.ts +117 -0
- package/src/core/ref-parser.ts +145 -0
- package/src/ingestion/frontmatter.ts +95 -2
- package/src/ingestion/sync.ts +281 -1
- package/src/mcp/tools/get.ts +1 -1
- package/src/mcp/tools/index.ts +89 -1
- package/src/mcp/tools/links.ts +83 -1
- package/src/mcp/tools/multi-get.ts +1 -1
- package/src/mcp/tools/query.ts +207 -0
- package/src/pipeline/diagnose.ts +302 -0
- package/src/pipeline/filters.ts +119 -0
- package/src/pipeline/hybrid.ts +90 -17
- package/src/pipeline/types.ts +32 -0
- package/src/publish/export-service.ts +1 -1
- package/src/sdk/documents.ts +2 -2
- package/src/serve/routes/api.ts +194 -0
- package/src/serve/routes/graph.ts +173 -1
- package/src/serve/server.ts +24 -1
- package/src/store/migrations/010-typed-edges.ts +67 -0
- package/src/store/migrations/011-doc-edge-traversal-indexes.ts +30 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +826 -102
- package/src/store/types.ts +141 -0
|
@@ -21,6 +21,11 @@ import type {
|
|
|
21
21
|
CleanupStats,
|
|
22
22
|
CollectionRow,
|
|
23
23
|
ContextRow,
|
|
24
|
+
DocEdgeConfidence,
|
|
25
|
+
DocEdgeInput,
|
|
26
|
+
DocEdgeRow,
|
|
27
|
+
DocEdgeSource,
|
|
28
|
+
DocEdgeType,
|
|
24
29
|
DocLinkInput,
|
|
25
30
|
DocLinkRow,
|
|
26
31
|
DocLinkSource,
|
|
@@ -33,6 +38,8 @@ import type {
|
|
|
33
38
|
GraphEdgeConfidence,
|
|
34
39
|
GraphEdgeAudit,
|
|
35
40
|
GraphLinkType,
|
|
41
|
+
GraphQueryOptions,
|
|
42
|
+
GraphQueryTraversalRows,
|
|
36
43
|
GraphReportNode,
|
|
37
44
|
GraphResult,
|
|
38
45
|
IndexStatus,
|
|
@@ -50,6 +57,11 @@ import type { SqliteDbProvider } from "./types";
|
|
|
50
57
|
|
|
51
58
|
import { buildUri, deriveDocid, stripUriIndex } from "../../app/constants";
|
|
52
59
|
import { analyzeGraphCommunities } from "../../core/graph-analysis";
|
|
60
|
+
import {
|
|
61
|
+
buildWikiBestMatchSubquery,
|
|
62
|
+
buildWikiBestRankMatchCountSubquery,
|
|
63
|
+
buildWikiBestRankSubquery,
|
|
64
|
+
} from "../../core/graph-resolver";
|
|
53
65
|
import { normalizeWikiName, stripWikiMdExt } from "../../core/links";
|
|
54
66
|
import { migrations, runMigrations } from "../migrations";
|
|
55
67
|
import { err, ok } from "../types";
|
|
@@ -65,6 +77,7 @@ import { loadFts5Snowball } from "./fts5-snowball";
|
|
|
65
77
|
const WHITESPACE_REGEX = /\s+/;
|
|
66
78
|
const SINGLE_LINE_QUERY_PATTERN = /[\r\n]/;
|
|
67
79
|
const DOUBLE_QUOTE_PATTERN = /"/g;
|
|
80
|
+
const DOC_EDGE_TYPE_PATTERN = /^[a-z][a-z0-9_]*$/;
|
|
68
81
|
const FTS5_FIELD_WEIGHTS = {
|
|
69
82
|
filepath: 1.5,
|
|
70
83
|
title: 4.0,
|
|
@@ -89,6 +102,16 @@ function sanitizeCompoundTerm(term: string): string {
|
|
|
89
102
|
.join(" ");
|
|
90
103
|
}
|
|
91
104
|
|
|
105
|
+
function normalizeDocEdgeType(edgeType: DocEdgeType): string {
|
|
106
|
+
const normalized = edgeType.trim().toLowerCase();
|
|
107
|
+
if (!DOC_EDGE_TYPE_PATTERN.test(normalized)) {
|
|
108
|
+
throw new Error(
|
|
109
|
+
`Invalid edge_type "${edgeType}" (expected lowercase snake_case)`
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
return normalized;
|
|
113
|
+
}
|
|
114
|
+
|
|
92
115
|
type FtsQueryBuildResult =
|
|
93
116
|
| { ok: true; query: string }
|
|
94
117
|
| { ok: false; error: string };
|
|
@@ -554,10 +577,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
554
577
|
collection, rel_path, source_hash, source_mime, source_ext,
|
|
555
578
|
source_size, source_mtime, source_ctime, docid, uri, title, mirror_hash,
|
|
556
579
|
converter_id, converter_version, language_hint, content_type, categories,
|
|
557
|
-
author, frontmatter_date, date_fields, content_type_rules_fingerprint,
|
|
580
|
+
content_type_source, author, frontmatter_date, date_fields, content_type_rules_fingerprint,
|
|
558
581
|
active, indexed_at, last_error_code, last_error_message, last_error_at,
|
|
559
582
|
ingest_version, updated_at
|
|
560
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'), ?, ?, ?, ?, datetime('now'))
|
|
583
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'), ?, ?, ?, ?, datetime('now'))
|
|
561
584
|
ON CONFLICT(collection, rel_path) DO UPDATE SET
|
|
562
585
|
source_hash = excluded.source_hash,
|
|
563
586
|
source_mime = excluded.source_mime,
|
|
@@ -574,6 +597,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
574
597
|
language_hint = excluded.language_hint,
|
|
575
598
|
content_type = excluded.content_type,
|
|
576
599
|
categories = excluded.categories,
|
|
600
|
+
content_type_source = excluded.content_type_source,
|
|
577
601
|
author = excluded.author,
|
|
578
602
|
frontmatter_date = excluded.frontmatter_date,
|
|
579
603
|
date_fields = excluded.date_fields,
|
|
@@ -604,6 +628,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
604
628
|
doc.languageHint ?? null,
|
|
605
629
|
doc.contentType ?? null,
|
|
606
630
|
doc.categories ? JSON.stringify(doc.categories) : null,
|
|
631
|
+
doc.contentTypeSource ?? null,
|
|
607
632
|
doc.author ?? null,
|
|
608
633
|
doc.frontmatterDate ?? null,
|
|
609
634
|
doc.dateFields ? JSON.stringify(doc.dateFields) : null,
|
|
@@ -2294,6 +2319,768 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2294
2319
|
}
|
|
2295
2320
|
}
|
|
2296
2321
|
|
|
2322
|
+
/**
|
|
2323
|
+
* Set semantic edges for a document.
|
|
2324
|
+
* Replaces edges from the given source.
|
|
2325
|
+
*/
|
|
2326
|
+
async setDocEdges(
|
|
2327
|
+
documentId: number,
|
|
2328
|
+
edges: DocEdgeInput[],
|
|
2329
|
+
source: DocEdgeSource
|
|
2330
|
+
): Promise<StoreResult<void>> {
|
|
2331
|
+
try {
|
|
2332
|
+
const db = this.ensureOpen();
|
|
2333
|
+
|
|
2334
|
+
const transaction = db.transaction(() => {
|
|
2335
|
+
db.run("DELETE FROM doc_edges WHERE src_doc_id = ? AND source = ?", [
|
|
2336
|
+
documentId,
|
|
2337
|
+
source,
|
|
2338
|
+
]);
|
|
2339
|
+
|
|
2340
|
+
if (edges.length === 0) {
|
|
2341
|
+
return;
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2344
|
+
const stmt = db.prepare(`
|
|
2345
|
+
INSERT INTO doc_edges (
|
|
2346
|
+
src_doc_id, dst_doc_id, edge_type, confidence, source
|
|
2347
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
2348
|
+
ON CONFLICT(src_doc_id, dst_doc_id, edge_type, source) DO UPDATE SET
|
|
2349
|
+
confidence = excluded.confidence
|
|
2350
|
+
`);
|
|
2351
|
+
|
|
2352
|
+
for (const edge of edges) {
|
|
2353
|
+
stmt.run(
|
|
2354
|
+
documentId,
|
|
2355
|
+
edge.targetDocId,
|
|
2356
|
+
normalizeDocEdgeType(edge.edgeType),
|
|
2357
|
+
edge.confidence,
|
|
2358
|
+
source
|
|
2359
|
+
);
|
|
2360
|
+
}
|
|
2361
|
+
});
|
|
2362
|
+
|
|
2363
|
+
transaction();
|
|
2364
|
+
return ok(undefined);
|
|
2365
|
+
} catch (cause) {
|
|
2366
|
+
return err(
|
|
2367
|
+
"QUERY_FAILED",
|
|
2368
|
+
cause instanceof Error ? cause.message : "Failed to set document edges",
|
|
2369
|
+
cause
|
|
2370
|
+
);
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
|
|
2374
|
+
async getEdgesForDoc(
|
|
2375
|
+
documentId: number,
|
|
2376
|
+
options?: { edgeType?: DocEdgeType }
|
|
2377
|
+
): Promise<StoreResult<DocEdgeRow[]>> {
|
|
2378
|
+
try {
|
|
2379
|
+
const db = this.ensureOpen();
|
|
2380
|
+
const params: (number | string)[] = [documentId];
|
|
2381
|
+
const edgeTypeClause = options?.edgeType ? "AND e.edge_type = ?" : "";
|
|
2382
|
+
if (options?.edgeType) {
|
|
2383
|
+
params.push(normalizeDocEdgeType(options.edgeType));
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2386
|
+
const rows = db
|
|
2387
|
+
.query<DbDocEdgeRow, (number | string)[]>(
|
|
2388
|
+
`
|
|
2389
|
+
WITH ranked AS (
|
|
2390
|
+
SELECT
|
|
2391
|
+
e.src_doc_id,
|
|
2392
|
+
src.docid AS source_docid,
|
|
2393
|
+
src.uri AS source_uri,
|
|
2394
|
+
src.title AS source_title,
|
|
2395
|
+
e.dst_doc_id,
|
|
2396
|
+
dst.docid AS target_docid,
|
|
2397
|
+
dst.uri AS target_uri,
|
|
2398
|
+
dst.title AS target_title,
|
|
2399
|
+
e.edge_type,
|
|
2400
|
+
e.confidence,
|
|
2401
|
+
e.source,
|
|
2402
|
+
ROW_NUMBER() OVER (
|
|
2403
|
+
PARTITION BY e.src_doc_id, e.dst_doc_id, e.edge_type
|
|
2404
|
+
ORDER BY
|
|
2405
|
+
CASE e.confidence
|
|
2406
|
+
WHEN 'manual' THEN 1
|
|
2407
|
+
WHEN 'configured' THEN 2
|
|
2408
|
+
WHEN 'parsed' THEN 3
|
|
2409
|
+
WHEN 'inferred' THEN 4
|
|
2410
|
+
ELSE 5
|
|
2411
|
+
END,
|
|
2412
|
+
e.source ASC,
|
|
2413
|
+
dst.docid ASC,
|
|
2414
|
+
dst.uri ASC
|
|
2415
|
+
) AS rn
|
|
2416
|
+
FROM doc_edges e
|
|
2417
|
+
JOIN documents src ON src.id = e.src_doc_id AND src.active = 1
|
|
2418
|
+
JOIN documents dst ON dst.id = e.dst_doc_id AND dst.active = 1
|
|
2419
|
+
WHERE e.src_doc_id = ?
|
|
2420
|
+
${edgeTypeClause}
|
|
2421
|
+
)
|
|
2422
|
+
SELECT *
|
|
2423
|
+
FROM ranked
|
|
2424
|
+
WHERE rn = 1
|
|
2425
|
+
ORDER BY edge_type ASC, target_uri ASC, target_docid ASC
|
|
2426
|
+
`
|
|
2427
|
+
)
|
|
2428
|
+
.all(...params);
|
|
2429
|
+
|
|
2430
|
+
return ok(rows.map(mapDocEdgeRow));
|
|
2431
|
+
} catch (cause) {
|
|
2432
|
+
return err(
|
|
2433
|
+
"QUERY_FAILED",
|
|
2434
|
+
cause instanceof Error
|
|
2435
|
+
? cause.message
|
|
2436
|
+
: "Failed to get edges for document",
|
|
2437
|
+
cause
|
|
2438
|
+
);
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
|
|
2442
|
+
async getEdgeBacklinksForDoc(
|
|
2443
|
+
documentId: number,
|
|
2444
|
+
options?: { collection?: string; edgeType?: DocEdgeType }
|
|
2445
|
+
): Promise<StoreResult<DocEdgeRow[]>> {
|
|
2446
|
+
try {
|
|
2447
|
+
const db = this.ensureOpen();
|
|
2448
|
+
const params: (number | string)[] = [documentId];
|
|
2449
|
+
const conditions: string[] = ["e.dst_doc_id = ?"];
|
|
2450
|
+
if (options?.collection) {
|
|
2451
|
+
conditions.push("src.collection = ?");
|
|
2452
|
+
params.push(options.collection);
|
|
2453
|
+
}
|
|
2454
|
+
if (options?.edgeType) {
|
|
2455
|
+
conditions.push("e.edge_type = ?");
|
|
2456
|
+
params.push(normalizeDocEdgeType(options.edgeType));
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
const rows = db
|
|
2460
|
+
.query<DbDocEdgeRow, (number | string)[]>(
|
|
2461
|
+
`
|
|
2462
|
+
WITH ranked AS (
|
|
2463
|
+
SELECT
|
|
2464
|
+
e.src_doc_id,
|
|
2465
|
+
src.docid AS source_docid,
|
|
2466
|
+
src.uri AS source_uri,
|
|
2467
|
+
src.title AS source_title,
|
|
2468
|
+
e.dst_doc_id,
|
|
2469
|
+
dst.docid AS target_docid,
|
|
2470
|
+
dst.uri AS target_uri,
|
|
2471
|
+
dst.title AS target_title,
|
|
2472
|
+
e.edge_type,
|
|
2473
|
+
e.confidence,
|
|
2474
|
+
e.source,
|
|
2475
|
+
ROW_NUMBER() OVER (
|
|
2476
|
+
PARTITION BY e.src_doc_id, e.dst_doc_id, e.edge_type
|
|
2477
|
+
ORDER BY
|
|
2478
|
+
CASE e.confidence
|
|
2479
|
+
WHEN 'manual' THEN 1
|
|
2480
|
+
WHEN 'configured' THEN 2
|
|
2481
|
+
WHEN 'parsed' THEN 3
|
|
2482
|
+
WHEN 'inferred' THEN 4
|
|
2483
|
+
ELSE 5
|
|
2484
|
+
END,
|
|
2485
|
+
e.source ASC,
|
|
2486
|
+
src.docid ASC,
|
|
2487
|
+
src.uri ASC
|
|
2488
|
+
) AS rn
|
|
2489
|
+
FROM doc_edges e
|
|
2490
|
+
JOIN documents src ON src.id = e.src_doc_id AND src.active = 1
|
|
2491
|
+
JOIN documents dst ON dst.id = e.dst_doc_id AND dst.active = 1
|
|
2492
|
+
WHERE ${conditions.join(" AND ")}
|
|
2493
|
+
)
|
|
2494
|
+
SELECT *
|
|
2495
|
+
FROM ranked
|
|
2496
|
+
WHERE rn = 1
|
|
2497
|
+
ORDER BY edge_type ASC, source_uri ASC, source_docid ASC
|
|
2498
|
+
`
|
|
2499
|
+
)
|
|
2500
|
+
.all(...params);
|
|
2501
|
+
|
|
2502
|
+
return ok(rows.map(mapDocEdgeRow));
|
|
2503
|
+
} catch (cause) {
|
|
2504
|
+
return err(
|
|
2505
|
+
"QUERY_FAILED",
|
|
2506
|
+
cause instanceof Error
|
|
2507
|
+
? cause.message
|
|
2508
|
+
: "Failed to get edge backlinks for document",
|
|
2509
|
+
cause
|
|
2510
|
+
);
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
|
|
2514
|
+
async queryGraphTraversal(
|
|
2515
|
+
rootDocumentId: number,
|
|
2516
|
+
options: GraphQueryOptions = {}
|
|
2517
|
+
): Promise<StoreResult<GraphQueryTraversalRows>> {
|
|
2518
|
+
try {
|
|
2519
|
+
const db = this.ensureOpen();
|
|
2520
|
+
const direction = options.direction ?? "both";
|
|
2521
|
+
const maxDepth = Math.max(1, Math.min(options.maxDepth ?? 2, 6));
|
|
2522
|
+
const maxNodes = Math.max(1, Math.min(options.maxNodes ?? 100, 1_000));
|
|
2523
|
+
const frontierLimit = Math.max(
|
|
2524
|
+
1,
|
|
2525
|
+
Math.min(options.frontierLimit ?? 100, 1_000)
|
|
2526
|
+
);
|
|
2527
|
+
const visitedLimit = Math.max(
|
|
2528
|
+
1,
|
|
2529
|
+
Math.min(options.visitedLimit ?? 500, 5_000)
|
|
2530
|
+
);
|
|
2531
|
+
const edgeType = options.edgeType
|
|
2532
|
+
? normalizeDocEdgeType(options.edgeType)
|
|
2533
|
+
: undefined;
|
|
2534
|
+
|
|
2535
|
+
const nodeLimit = Math.min(maxNodes, visitedLimit);
|
|
2536
|
+
const edgeTypeFilter = edgeType ? "AND e.edge_type = ?" : "";
|
|
2537
|
+
const edgeTypeFilterFor = (alias: string): string =>
|
|
2538
|
+
edgeType ? `AND ${alias}.edge_type = ?` : "";
|
|
2539
|
+
const candidateEdgeLimit = Math.min(frontierLimit * 4, 4_000);
|
|
2540
|
+
const frontierCtes: string[] = [];
|
|
2541
|
+
const frontierParams: (number | string)[] = [];
|
|
2542
|
+
const frontierNames = ["f0"];
|
|
2543
|
+
const allFrontiers = (): string =>
|
|
2544
|
+
frontierNames
|
|
2545
|
+
.map((name) => `SELECT doc_id FROM ${name}`)
|
|
2546
|
+
.join(" UNION ");
|
|
2547
|
+
const frontierJoinClause = (frontierName: string): string =>
|
|
2548
|
+
direction === "out"
|
|
2549
|
+
? `e.src_doc_id = ${frontierName}.doc_id`
|
|
2550
|
+
: direction === "in"
|
|
2551
|
+
? `e.dst_doc_id = ${frontierName}.doc_id`
|
|
2552
|
+
: `(e.src_doc_id = ${frontierName}.doc_id OR e.dst_doc_id = ${frontierName}.doc_id)`;
|
|
2553
|
+
const boundedEdgePredicate = (frontierName: string): string => {
|
|
2554
|
+
if (direction === "out") {
|
|
2555
|
+
return `
|
|
2556
|
+
e.id IN (
|
|
2557
|
+
SELECT edge_id
|
|
2558
|
+
FROM (
|
|
2559
|
+
SELECT
|
|
2560
|
+
edge_id,
|
|
2561
|
+
edge_type,
|
|
2562
|
+
next_doc_id
|
|
2563
|
+
FROM (
|
|
2564
|
+
SELECT
|
|
2565
|
+
e2.id AS edge_id,
|
|
2566
|
+
e2.edge_type,
|
|
2567
|
+
e2.dst_doc_id AS next_doc_id,
|
|
2568
|
+
row_number() OVER (
|
|
2569
|
+
PARTITION BY e2.dst_doc_id
|
|
2570
|
+
ORDER BY e2.edge_type ASC, e2.id ASC
|
|
2571
|
+
) AS next_rank
|
|
2572
|
+
FROM doc_edges e2
|
|
2573
|
+
JOIN documents next_doc ON next_doc.id = e2.dst_doc_id AND next_doc.active = 1
|
|
2574
|
+
WHERE e2.src_doc_id = ${frontierName}.doc_id
|
|
2575
|
+
${edgeTypeFilterFor("e2")}
|
|
2576
|
+
AND instr(${frontierName}.path, printf(',%d,', e2.dst_doc_id)) = 0
|
|
2577
|
+
AND e2.dst_doc_id NOT IN (${allFrontiers()})
|
|
2578
|
+
)
|
|
2579
|
+
WHERE next_rank = 1
|
|
2580
|
+
ORDER BY edge_type ASC, next_doc_id ASC, edge_id ASC
|
|
2581
|
+
LIMIT ?
|
|
2582
|
+
)
|
|
2583
|
+
)`;
|
|
2584
|
+
}
|
|
2585
|
+
if (direction === "in") {
|
|
2586
|
+
return `
|
|
2587
|
+
e.id IN (
|
|
2588
|
+
SELECT edge_id
|
|
2589
|
+
FROM (
|
|
2590
|
+
SELECT
|
|
2591
|
+
edge_id,
|
|
2592
|
+
edge_type,
|
|
2593
|
+
next_doc_id
|
|
2594
|
+
FROM (
|
|
2595
|
+
SELECT
|
|
2596
|
+
e2.id AS edge_id,
|
|
2597
|
+
e2.edge_type,
|
|
2598
|
+
e2.src_doc_id AS next_doc_id,
|
|
2599
|
+
row_number() OVER (
|
|
2600
|
+
PARTITION BY e2.src_doc_id
|
|
2601
|
+
ORDER BY e2.edge_type ASC, e2.id ASC
|
|
2602
|
+
) AS next_rank
|
|
2603
|
+
FROM doc_edges e2
|
|
2604
|
+
JOIN documents next_doc ON next_doc.id = e2.src_doc_id AND next_doc.active = 1
|
|
2605
|
+
WHERE e2.dst_doc_id = ${frontierName}.doc_id
|
|
2606
|
+
${edgeTypeFilterFor("e2")}
|
|
2607
|
+
AND instr(${frontierName}.path, printf(',%d,', e2.src_doc_id)) = 0
|
|
2608
|
+
AND e2.src_doc_id NOT IN (${allFrontiers()})
|
|
2609
|
+
)
|
|
2610
|
+
WHERE next_rank = 1
|
|
2611
|
+
ORDER BY edge_type ASC, next_doc_id ASC, edge_id ASC
|
|
2612
|
+
LIMIT ?
|
|
2613
|
+
)
|
|
2614
|
+
)`;
|
|
2615
|
+
}
|
|
2616
|
+
return `
|
|
2617
|
+
(
|
|
2618
|
+
e.id IN (
|
|
2619
|
+
SELECT edge_id
|
|
2620
|
+
FROM (
|
|
2621
|
+
SELECT
|
|
2622
|
+
edge_id,
|
|
2623
|
+
edge_type,
|
|
2624
|
+
next_doc_id
|
|
2625
|
+
FROM (
|
|
2626
|
+
SELECT
|
|
2627
|
+
e2.id AS edge_id,
|
|
2628
|
+
e2.edge_type,
|
|
2629
|
+
e2.dst_doc_id AS next_doc_id,
|
|
2630
|
+
row_number() OVER (
|
|
2631
|
+
PARTITION BY e2.dst_doc_id
|
|
2632
|
+
ORDER BY e2.edge_type ASC, e2.id ASC
|
|
2633
|
+
) AS next_rank
|
|
2634
|
+
FROM doc_edges e2
|
|
2635
|
+
JOIN documents next_doc ON next_doc.id = e2.dst_doc_id AND next_doc.active = 1
|
|
2636
|
+
WHERE e2.src_doc_id = ${frontierName}.doc_id
|
|
2637
|
+
${edgeTypeFilterFor("e2")}
|
|
2638
|
+
AND instr(${frontierName}.path, printf(',%d,', e2.dst_doc_id)) = 0
|
|
2639
|
+
AND e2.dst_doc_id NOT IN (${allFrontiers()})
|
|
2640
|
+
)
|
|
2641
|
+
WHERE next_rank = 1
|
|
2642
|
+
ORDER BY edge_type ASC, next_doc_id ASC, edge_id ASC
|
|
2643
|
+
LIMIT ?
|
|
2644
|
+
)
|
|
2645
|
+
)
|
|
2646
|
+
OR e.id IN (
|
|
2647
|
+
SELECT edge_id
|
|
2648
|
+
FROM (
|
|
2649
|
+
SELECT
|
|
2650
|
+
edge_id,
|
|
2651
|
+
edge_type,
|
|
2652
|
+
next_doc_id
|
|
2653
|
+
FROM (
|
|
2654
|
+
SELECT
|
|
2655
|
+
e3.id AS edge_id,
|
|
2656
|
+
e3.edge_type,
|
|
2657
|
+
e3.src_doc_id AS next_doc_id,
|
|
2658
|
+
row_number() OVER (
|
|
2659
|
+
PARTITION BY e3.src_doc_id
|
|
2660
|
+
ORDER BY e3.edge_type ASC, e3.id ASC
|
|
2661
|
+
) AS next_rank
|
|
2662
|
+
FROM doc_edges e3
|
|
2663
|
+
JOIN documents next_doc ON next_doc.id = e3.src_doc_id AND next_doc.active = 1
|
|
2664
|
+
WHERE e3.dst_doc_id = ${frontierName}.doc_id
|
|
2665
|
+
${edgeTypeFilterFor("e3")}
|
|
2666
|
+
AND instr(${frontierName}.path, printf(',%d,', e3.src_doc_id)) = 0
|
|
2667
|
+
AND e3.src_doc_id NOT IN (${allFrontiers()})
|
|
2668
|
+
)
|
|
2669
|
+
WHERE next_rank = 1
|
|
2670
|
+
ORDER BY edge_type ASC, next_doc_id ASC, edge_id ASC
|
|
2671
|
+
LIMIT ?
|
|
2672
|
+
)
|
|
2673
|
+
)
|
|
2674
|
+
)`;
|
|
2675
|
+
};
|
|
2676
|
+
const nextExprFor = (frontierName: string): string =>
|
|
2677
|
+
`CASE WHEN e.src_doc_id = ${frontierName}.doc_id THEN e.dst_doc_id ELSE e.src_doc_id END`;
|
|
2678
|
+
|
|
2679
|
+
const boundaryCandidateDepth = maxDepth + 1;
|
|
2680
|
+
for (let depth = 1; depth <= boundaryCandidateDepth; depth += 1) {
|
|
2681
|
+
const previous = `f${depth - 1}`;
|
|
2682
|
+
const raw = `c${depth}_raw`;
|
|
2683
|
+
const deduped = `c${depth}_deduped`;
|
|
2684
|
+
const nodeDeduped = `c${depth}_node_deduped`;
|
|
2685
|
+
const ranked = `c${depth}_ranked`;
|
|
2686
|
+
const candidates = `c${depth}`;
|
|
2687
|
+
const frontier = `f${depth}`;
|
|
2688
|
+
const nextExpr = nextExprFor(previous);
|
|
2689
|
+
frontierCtes.push(`
|
|
2690
|
+
${raw} AS (
|
|
2691
|
+
SELECT
|
|
2692
|
+
${previous}.doc_id AS from_doc_id,
|
|
2693
|
+
${nextExpr} AS doc_id,
|
|
2694
|
+
${depth} AS depth,
|
|
2695
|
+
${previous}.path || ${nextExpr} || ',' AS path,
|
|
2696
|
+
printf('%06d|%s|%012d', ${depth}, e.edge_type, ${nextExpr}) AS sort_key,
|
|
2697
|
+
e.src_doc_id,
|
|
2698
|
+
src.docid AS source_docid,
|
|
2699
|
+
src.uri AS source_uri,
|
|
2700
|
+
src.title AS source_title,
|
|
2701
|
+
e.dst_doc_id,
|
|
2702
|
+
dst.docid AS target_docid,
|
|
2703
|
+
dst.uri AS target_uri,
|
|
2704
|
+
dst.title AS target_title,
|
|
2705
|
+
e.edge_type,
|
|
2706
|
+
e.confidence,
|
|
2707
|
+
e.source,
|
|
2708
|
+
row_number() OVER (
|
|
2709
|
+
PARTITION BY ${previous}.doc_id, e.src_doc_id, e.dst_doc_id, e.edge_type
|
|
2710
|
+
ORDER BY
|
|
2711
|
+
CASE e.confidence
|
|
2712
|
+
WHEN 'manual' THEN 1
|
|
2713
|
+
WHEN 'configured' THEN 2
|
|
2714
|
+
WHEN 'parsed' THEN 3
|
|
2715
|
+
WHEN 'inferred' THEN 4
|
|
2716
|
+
ELSE 5
|
|
2717
|
+
END,
|
|
2718
|
+
e.source ASC,
|
|
2719
|
+
src.docid ASC,
|
|
2720
|
+
dst.docid ASC
|
|
2721
|
+
) AS dedup_rank
|
|
2722
|
+
FROM ${previous}
|
|
2723
|
+
JOIN doc_edges e ON ${frontierJoinClause(previous)}
|
|
2724
|
+
AND ${boundedEdgePredicate(previous)}
|
|
2725
|
+
JOIN documents src ON src.id = e.src_doc_id AND src.active = 1
|
|
2726
|
+
JOIN documents dst ON dst.id = e.dst_doc_id AND dst.active = 1
|
|
2727
|
+
WHERE 1 = 1
|
|
2728
|
+
${edgeTypeFilter}
|
|
2729
|
+
AND instr(${previous}.path, printf(',%d,', ${nextExpr})) = 0
|
|
2730
|
+
AND ${nextExpr} NOT IN (${allFrontiers()})
|
|
2731
|
+
),
|
|
2732
|
+
${deduped} AS (
|
|
2733
|
+
SELECT *
|
|
2734
|
+
FROM ${raw}
|
|
2735
|
+
WHERE dedup_rank = 1
|
|
2736
|
+
),
|
|
2737
|
+
${nodeDeduped} AS (
|
|
2738
|
+
SELECT
|
|
2739
|
+
from_doc_id,
|
|
2740
|
+
doc_id,
|
|
2741
|
+
depth,
|
|
2742
|
+
min(path) AS path,
|
|
2743
|
+
min(sort_key) AS sort_key
|
|
2744
|
+
FROM ${deduped}
|
|
2745
|
+
GROUP BY from_doc_id, doc_id, depth
|
|
2746
|
+
),
|
|
2747
|
+
${ranked} AS (
|
|
2748
|
+
SELECT
|
|
2749
|
+
*,
|
|
2750
|
+
row_number() OVER (
|
|
2751
|
+
PARTITION BY from_doc_id
|
|
2752
|
+
ORDER BY sort_key ASC, doc_id ASC
|
|
2753
|
+
) AS expansion_rank
|
|
2754
|
+
FROM ${nodeDeduped}
|
|
2755
|
+
),
|
|
2756
|
+
${candidates} AS (
|
|
2757
|
+
SELECT doc_id, depth, min(path) AS path, min(sort_key) AS sort_key
|
|
2758
|
+
FROM ${ranked}
|
|
2759
|
+
WHERE expansion_rank <= ?
|
|
2760
|
+
GROUP BY doc_id, depth
|
|
2761
|
+
),
|
|
2762
|
+
${frontier} AS (
|
|
2763
|
+
SELECT doc_id, depth, path, sort_key
|
|
2764
|
+
FROM (
|
|
2765
|
+
SELECT
|
|
2766
|
+
doc_id,
|
|
2767
|
+
depth,
|
|
2768
|
+
path,
|
|
2769
|
+
sort_key,
|
|
2770
|
+
row_number() OVER (ORDER BY sort_key ASC, doc_id ASC) AS depth_rank
|
|
2771
|
+
FROM ${candidates}
|
|
2772
|
+
)
|
|
2773
|
+
WHERE depth_rank <= ?
|
|
2774
|
+
AND depth_rank <= ? - (
|
|
2775
|
+
SELECT count(*)
|
|
2776
|
+
FROM (${allFrontiers()})
|
|
2777
|
+
)
|
|
2778
|
+
)`);
|
|
2779
|
+
if (direction === "both") {
|
|
2780
|
+
if (edgeType) {
|
|
2781
|
+
frontierParams.push(edgeType);
|
|
2782
|
+
}
|
|
2783
|
+
frontierParams.push(candidateEdgeLimit);
|
|
2784
|
+
if (edgeType) {
|
|
2785
|
+
frontierParams.push(edgeType);
|
|
2786
|
+
}
|
|
2787
|
+
frontierParams.push(candidateEdgeLimit);
|
|
2788
|
+
} else {
|
|
2789
|
+
if (edgeType) {
|
|
2790
|
+
frontierParams.push(edgeType);
|
|
2791
|
+
}
|
|
2792
|
+
frontierParams.push(candidateEdgeLimit);
|
|
2793
|
+
}
|
|
2794
|
+
if (edgeType) {
|
|
2795
|
+
frontierParams.push(edgeType);
|
|
2796
|
+
}
|
|
2797
|
+
frontierParams.push(frontierLimit, frontierLimit, visitedLimit);
|
|
2798
|
+
if (depth <= maxDepth) {
|
|
2799
|
+
frontierNames.push(frontier);
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2802
|
+
|
|
2803
|
+
const returnedEdgeDirectionClause =
|
|
2804
|
+
direction === "out"
|
|
2805
|
+
? "AND ns.depth < nt.depth"
|
|
2806
|
+
: direction === "in"
|
|
2807
|
+
? "AND ns.depth > nt.depth"
|
|
2808
|
+
: "";
|
|
2809
|
+
const frontierUnion = frontierNames
|
|
2810
|
+
.map((name) => `SELECT doc_id, depth, sort_key FROM ${name}`)
|
|
2811
|
+
.join(" UNION ALL ");
|
|
2812
|
+
const visitedOverflowChecks = Array.from(
|
|
2813
|
+
{ length: maxDepth },
|
|
2814
|
+
(_, index) => {
|
|
2815
|
+
const previousFrontiers = Array.from(
|
|
2816
|
+
{ length: index + 1 },
|
|
2817
|
+
(__, frontierIndex) => `SELECT doc_id FROM f${frontierIndex}`
|
|
2818
|
+
).join(" UNION ");
|
|
2819
|
+
return `
|
|
2820
|
+
SELECT 1 AS has_more
|
|
2821
|
+
FROM c${index + 1}
|
|
2822
|
+
GROUP BY 1
|
|
2823
|
+
HAVING count(*) > ? - (
|
|
2824
|
+
SELECT count(*)
|
|
2825
|
+
FROM (${previousFrontiers})
|
|
2826
|
+
)`;
|
|
2827
|
+
}
|
|
2828
|
+
).join(" UNION ALL ");
|
|
2829
|
+
const frontierOverflowChecks = Array.from(
|
|
2830
|
+
{ length: maxDepth },
|
|
2831
|
+
(_, index) => `
|
|
2832
|
+
SELECT 1 AS has_more
|
|
2833
|
+
FROM c${index + 1}
|
|
2834
|
+
GROUP BY 1
|
|
2835
|
+
HAVING count(*) > ?
|
|
2836
|
+
UNION ALL
|
|
2837
|
+
SELECT 1 AS has_more
|
|
2838
|
+
FROM c${index + 1}_ranked
|
|
2839
|
+
WHERE expansion_rank > ?`
|
|
2840
|
+
).join(" UNION ALL ");
|
|
2841
|
+
const walkCte = `
|
|
2842
|
+
WITH RECURSIVE
|
|
2843
|
+
f0(doc_id, depth, path, sort_key) AS (
|
|
2844
|
+
SELECT ?, 0, printf(',%d,', ?), ''
|
|
2845
|
+
),
|
|
2846
|
+
${frontierCtes.join(",")},
|
|
2847
|
+
node_depth AS (
|
|
2848
|
+
SELECT doc_id, min(depth) AS depth, min(sort_key) AS sort_key
|
|
2849
|
+
FROM (${frontierUnion})
|
|
2850
|
+
GROUP BY doc_id
|
|
2851
|
+
),
|
|
2852
|
+
ranked_nodes AS (
|
|
2853
|
+
SELECT
|
|
2854
|
+
d.*,
|
|
2855
|
+
nd.depth,
|
|
2856
|
+
row_number() OVER (ORDER BY nd.depth ASC, nd.sort_key ASC, d.id ASC) AS global_rank,
|
|
2857
|
+
count(*) OVER () AS total_count
|
|
2858
|
+
FROM node_depth nd
|
|
2859
|
+
JOIN documents d ON d.id = nd.doc_id AND d.active = 1
|
|
2860
|
+
)
|
|
2861
|
+
`;
|
|
2862
|
+
|
|
2863
|
+
const baseParams: (number | string)[] = [];
|
|
2864
|
+
baseParams.push(rootDocumentId, rootDocumentId, ...frontierParams);
|
|
2865
|
+
|
|
2866
|
+
const nodeRows = db
|
|
2867
|
+
.query<
|
|
2868
|
+
DbDocumentRow & {
|
|
2869
|
+
depth: number;
|
|
2870
|
+
global_rank: number;
|
|
2871
|
+
total_count: number;
|
|
2872
|
+
},
|
|
2873
|
+
(number | string)[]
|
|
2874
|
+
>(
|
|
2875
|
+
`${walkCte}
|
|
2876
|
+
SELECT *
|
|
2877
|
+
FROM ranked_nodes
|
|
2878
|
+
WHERE global_rank <= ?
|
|
2879
|
+
ORDER BY depth ASC, uri ASC, docid ASC
|
|
2880
|
+
LIMIT ?`
|
|
2881
|
+
)
|
|
2882
|
+
.all(...baseParams, nodeLimit + 1, nodeLimit + 1);
|
|
2883
|
+
|
|
2884
|
+
const returnedNodeRows = nodeRows.slice(0, nodeLimit);
|
|
2885
|
+
const returnedIds = new Set(returnedNodeRows.map((row) => row.id));
|
|
2886
|
+
const totalNodes = nodeRows[0]?.total_count ?? 0;
|
|
2887
|
+
const warnings: string[] = [];
|
|
2888
|
+
let truncated = false;
|
|
2889
|
+
if (totalNodes > maxNodes) {
|
|
2890
|
+
truncated = true;
|
|
2891
|
+
warnings.push("maxNodes reached");
|
|
2892
|
+
}
|
|
2893
|
+
if (totalNodes > visitedLimit) {
|
|
2894
|
+
truncated = true;
|
|
2895
|
+
warnings.push("visitedLimit reached");
|
|
2896
|
+
}
|
|
2897
|
+
|
|
2898
|
+
const visitedRows = db
|
|
2899
|
+
.query<{ has_more: number }, (number | string)[]>(
|
|
2900
|
+
`${walkCte}
|
|
2901
|
+
SELECT 1 AS has_more
|
|
2902
|
+
FROM (${visitedOverflowChecks})
|
|
2903
|
+
LIMIT 1`
|
|
2904
|
+
)
|
|
2905
|
+
.get(...baseParams, ...Array(maxDepth).fill(visitedLimit));
|
|
2906
|
+
if (visitedRows) {
|
|
2907
|
+
truncated = true;
|
|
2908
|
+
warnings.push("visitedLimit reached");
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
const frontierRows = db
|
|
2912
|
+
.query<{ has_more: number }, (number | string)[]>(
|
|
2913
|
+
`${walkCte}
|
|
2914
|
+
SELECT 1 AS has_more
|
|
2915
|
+
FROM (${frontierOverflowChecks})
|
|
2916
|
+
LIMIT 1`
|
|
2917
|
+
)
|
|
2918
|
+
.get(...baseParams, ...Array(maxDepth * 2).fill(frontierLimit));
|
|
2919
|
+
if (frontierRows) {
|
|
2920
|
+
truncated = true;
|
|
2921
|
+
warnings.push("frontierLimit reached");
|
|
2922
|
+
}
|
|
2923
|
+
|
|
2924
|
+
const edgeRows = db
|
|
2925
|
+
.query<DbDocEdgeRow & { traversal_depth: number }, (number | string)[]>(
|
|
2926
|
+
`${walkCte}
|
|
2927
|
+
, returned_edges AS (
|
|
2928
|
+
SELECT
|
|
2929
|
+
e.src_doc_id,
|
|
2930
|
+
src.docid AS source_docid,
|
|
2931
|
+
src.uri AS source_uri,
|
|
2932
|
+
src.title AS source_title,
|
|
2933
|
+
e.dst_doc_id,
|
|
2934
|
+
dst.docid AS target_docid,
|
|
2935
|
+
dst.uri AS target_uri,
|
|
2936
|
+
dst.title AS target_title,
|
|
2937
|
+
e.edge_type,
|
|
2938
|
+
e.confidence,
|
|
2939
|
+
e.source,
|
|
2940
|
+
CASE
|
|
2941
|
+
WHEN ns.depth >= nt.depth THEN ns.depth
|
|
2942
|
+
ELSE nt.depth
|
|
2943
|
+
END AS traversal_depth,
|
|
2944
|
+
row_number() OVER (
|
|
2945
|
+
PARTITION BY e.src_doc_id, e.dst_doc_id, e.edge_type
|
|
2946
|
+
ORDER BY
|
|
2947
|
+
CASE e.confidence
|
|
2948
|
+
WHEN 'manual' THEN 1
|
|
2949
|
+
WHEN 'configured' THEN 2
|
|
2950
|
+
WHEN 'parsed' THEN 3
|
|
2951
|
+
WHEN 'inferred' THEN 4
|
|
2952
|
+
ELSE 5
|
|
2953
|
+
END,
|
|
2954
|
+
e.source ASC,
|
|
2955
|
+
src.docid ASC,
|
|
2956
|
+
dst.docid ASC
|
|
2957
|
+
) AS dedup_rank
|
|
2958
|
+
FROM doc_edges e
|
|
2959
|
+
JOIN node_depth ns ON ns.doc_id = e.src_doc_id
|
|
2960
|
+
JOIN node_depth nt ON nt.doc_id = e.dst_doc_id
|
|
2961
|
+
JOIN documents src ON src.id = e.src_doc_id AND src.active = 1
|
|
2962
|
+
JOIN documents dst ON dst.id = e.dst_doc_id AND dst.active = 1
|
|
2963
|
+
WHERE ns.doc_id IN (${[...returnedIds].map(() => "?").join(",")})
|
|
2964
|
+
AND nt.doc_id IN (${[...returnedIds].map(() => "?").join(",")})
|
|
2965
|
+
${edgeTypeFilter}
|
|
2966
|
+
${returnedEdgeDirectionClause}
|
|
2967
|
+
)
|
|
2968
|
+
SELECT *
|
|
2969
|
+
FROM returned_edges
|
|
2970
|
+
WHERE dedup_rank = 1
|
|
2971
|
+
ORDER BY traversal_depth ASC, edge_type ASC, source_docid ASC, target_docid ASC`
|
|
2972
|
+
)
|
|
2973
|
+
.all(
|
|
2974
|
+
...baseParams,
|
|
2975
|
+
...returnedIds,
|
|
2976
|
+
...returnedIds,
|
|
2977
|
+
...(edgeType ? [edgeType] : [])
|
|
2978
|
+
);
|
|
2979
|
+
|
|
2980
|
+
const boundaryRows = db
|
|
2981
|
+
.query<{ has_more: number }, (number | string)[]>(
|
|
2982
|
+
`${walkCte}
|
|
2983
|
+
SELECT 1 AS has_more
|
|
2984
|
+
FROM c${boundaryCandidateDepth}_ranked
|
|
2985
|
+
LIMIT 1`
|
|
2986
|
+
)
|
|
2987
|
+
.get(...baseParams);
|
|
2988
|
+
|
|
2989
|
+
if (boundaryRows) {
|
|
2990
|
+
truncated = true;
|
|
2991
|
+
warnings.push("maxDepth reached");
|
|
2992
|
+
}
|
|
2993
|
+
|
|
2994
|
+
return ok({
|
|
2995
|
+
nodes: returnedNodeRows.map((row) => ({
|
|
2996
|
+
doc: mapDocumentRow(row),
|
|
2997
|
+
depth: row.depth,
|
|
2998
|
+
})),
|
|
2999
|
+
edges: edgeRows.map((row) => ({
|
|
3000
|
+
edge: mapDocEdgeRow(row),
|
|
3001
|
+
depth: row.traversal_depth,
|
|
3002
|
+
})),
|
|
3003
|
+
truncated,
|
|
3004
|
+
warnings: [...new Set(warnings)],
|
|
3005
|
+
});
|
|
3006
|
+
} catch (cause) {
|
|
3007
|
+
return err(
|
|
3008
|
+
"QUERY_FAILED",
|
|
3009
|
+
cause instanceof Error
|
|
3010
|
+
? cause.message
|
|
3011
|
+
: "Failed to run graph traversal",
|
|
3012
|
+
cause
|
|
3013
|
+
);
|
|
3014
|
+
}
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
async backfillDocEdges(): Promise<StoreResult<{ inserted: number }>> {
|
|
3018
|
+
try {
|
|
3019
|
+
const db = this.ensureOpen();
|
|
3020
|
+
let inserted = 0;
|
|
3021
|
+
|
|
3022
|
+
const transaction = db.transaction(() => {
|
|
3023
|
+
db.run("DELETE FROM doc_edges WHERE source IN (?, ?)", [
|
|
3024
|
+
"wikilink",
|
|
3025
|
+
"markdown-link",
|
|
3026
|
+
]);
|
|
3027
|
+
|
|
3028
|
+
const insertWiki = db.run(`
|
|
3029
|
+
INSERT OR IGNORE INTO doc_edges (
|
|
3030
|
+
src_doc_id, dst_doc_id, edge_type, confidence, source
|
|
3031
|
+
)
|
|
3032
|
+
SELECT
|
|
3033
|
+
src.id,
|
|
3034
|
+
tgt.id,
|
|
3035
|
+
'mentions',
|
|
3036
|
+
'parsed',
|
|
3037
|
+
'wikilink'
|
|
3038
|
+
FROM documents src
|
|
3039
|
+
JOIN doc_links dl ON dl.source_doc_id = src.id
|
|
3040
|
+
JOIN documents tgt ON tgt.id = (${buildWikiBestMatchSubquery(
|
|
3041
|
+
"COALESCE(dl.target_collection, src.collection)",
|
|
3042
|
+
"dl.target_ref_norm"
|
|
3043
|
+
)})
|
|
3044
|
+
WHERE src.active = 1
|
|
3045
|
+
AND tgt.active = 1
|
|
3046
|
+
AND dl.link_type = 'wiki'
|
|
3047
|
+
`);
|
|
3048
|
+
|
|
3049
|
+
const insertMarkdown = db.run(`
|
|
3050
|
+
INSERT OR IGNORE INTO doc_edges (
|
|
3051
|
+
src_doc_id, dst_doc_id, edge_type, confidence, source
|
|
3052
|
+
)
|
|
3053
|
+
SELECT
|
|
3054
|
+
src.id,
|
|
3055
|
+
tgt.id,
|
|
3056
|
+
'related_to',
|
|
3057
|
+
'parsed',
|
|
3058
|
+
'markdown-link'
|
|
3059
|
+
FROM documents src
|
|
3060
|
+
JOIN doc_links dl ON dl.source_doc_id = src.id
|
|
3061
|
+
JOIN documents tgt ON tgt.active = 1
|
|
3062
|
+
AND tgt.collection = COALESCE(dl.target_collection, src.collection)
|
|
3063
|
+
AND tgt.rel_path = dl.target_ref_norm
|
|
3064
|
+
WHERE src.active = 1
|
|
3065
|
+
AND dl.link_type = 'markdown'
|
|
3066
|
+
`);
|
|
3067
|
+
|
|
3068
|
+
inserted = insertWiki.changes + insertMarkdown.changes;
|
|
3069
|
+
});
|
|
3070
|
+
|
|
3071
|
+
transaction();
|
|
3072
|
+
return ok({ inserted });
|
|
3073
|
+
} catch (cause) {
|
|
3074
|
+
return err(
|
|
3075
|
+
"QUERY_FAILED",
|
|
3076
|
+
cause instanceof Error
|
|
3077
|
+
? cause.message
|
|
3078
|
+
: "Failed to backfill document edges",
|
|
3079
|
+
cause
|
|
3080
|
+
);
|
|
3081
|
+
}
|
|
3082
|
+
}
|
|
3083
|
+
|
|
2297
3084
|
// ─────────────────────────────────────────────────────────────────────────
|
|
2298
3085
|
// Graph
|
|
2299
3086
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -2329,102 +3116,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2329
3116
|
// sqlite-vec not loaded
|
|
2330
3117
|
}
|
|
2331
3118
|
|
|
2332
|
-
const wikiTitleExpr = (alias: string): string =>
|
|
2333
|
-
`lower(trim(${alias}.title))`;
|
|
2334
|
-
|
|
2335
|
-
const wikiRelPathExpr = (alias: string): string =>
|
|
2336
|
-
`lower(${alias}.rel_path)`;
|
|
2337
|
-
|
|
2338
|
-
const suffixMatch = (targetExpr: string, valueExpr: string): string =>
|
|
2339
|
-
`(substr(${targetExpr}, -length(${valueExpr})) = ${valueExpr}
|
|
2340
|
-
AND (length(${targetExpr}) = length(${valueExpr})
|
|
2341
|
-
OR substr(${targetExpr}, -length(${valueExpr}) - 1, 1) = '/'))`;
|
|
2342
|
-
|
|
2343
|
-
const wikiMatch = (alias: string, targetRefExpr: string): string => {
|
|
2344
|
-
const titleExpr = wikiTitleExpr(alias);
|
|
2345
|
-
const relExpr = wikiRelPathExpr(alias);
|
|
2346
|
-
const targetBaseExpr = `CASE
|
|
2347
|
-
WHEN ${targetRefExpr} LIKE '%.md' THEN substr(${targetRefExpr}, 1, length(${targetRefExpr}) - 3)
|
|
2348
|
-
ELSE ${targetRefExpr}
|
|
2349
|
-
END`;
|
|
2350
|
-
const targetMdExpr = `${targetBaseExpr} || '.md'`;
|
|
2351
|
-
return `(
|
|
2352
|
-
${titleExpr} = ${targetBaseExpr}
|
|
2353
|
-
OR ${titleExpr} = ${targetMdExpr}
|
|
2354
|
-
OR ${suffixMatch(targetBaseExpr, titleExpr)}
|
|
2355
|
-
OR ${suffixMatch(targetMdExpr, `${titleExpr} || '.md'`)}
|
|
2356
|
-
OR ${relExpr} = ${targetBaseExpr}
|
|
2357
|
-
OR ${relExpr} = ${targetMdExpr}
|
|
2358
|
-
OR ${suffixMatch(relExpr, targetMdExpr)}
|
|
2359
|
-
OR ${suffixMatch(relExpr, targetBaseExpr)}
|
|
2360
|
-
OR ${suffixMatch(targetMdExpr, relExpr)}
|
|
2361
|
-
OR ${suffixMatch(targetBaseExpr, relExpr)}
|
|
2362
|
-
)`;
|
|
2363
|
-
};
|
|
2364
|
-
|
|
2365
|
-
const wikiOrder = (alias: string, targetRefExpr: string): string => {
|
|
2366
|
-
const titleExpr = wikiTitleExpr(alias);
|
|
2367
|
-
const relExpr = wikiRelPathExpr(alias);
|
|
2368
|
-
const targetBaseExpr = `CASE
|
|
2369
|
-
WHEN ${targetRefExpr} LIKE '%.md' THEN substr(${targetRefExpr}, 1, length(${targetRefExpr}) - 3)
|
|
2370
|
-
ELSE ${targetRefExpr}
|
|
2371
|
-
END`;
|
|
2372
|
-
const targetMdExpr = `${targetBaseExpr} || '.md'`;
|
|
2373
|
-
return `CASE
|
|
2374
|
-
WHEN ${titleExpr} = ${targetBaseExpr} THEN 1
|
|
2375
|
-
WHEN ${titleExpr} = ${targetMdExpr} THEN 2
|
|
2376
|
-
WHEN ${suffixMatch(targetBaseExpr, titleExpr)} THEN 3
|
|
2377
|
-
WHEN ${suffixMatch(targetMdExpr, `${titleExpr} || '.md'`)} THEN 4
|
|
2378
|
-
WHEN ${relExpr} = ${targetBaseExpr} THEN 5
|
|
2379
|
-
WHEN ${relExpr} = ${targetMdExpr} THEN 6
|
|
2380
|
-
WHEN ${suffixMatch(relExpr, targetMdExpr)} THEN 7
|
|
2381
|
-
WHEN ${suffixMatch(relExpr, targetBaseExpr)} THEN 8
|
|
2382
|
-
WHEN ${suffixMatch(targetMdExpr, relExpr)} THEN 9
|
|
2383
|
-
WHEN ${suffixMatch(targetBaseExpr, relExpr)} THEN 10
|
|
2384
|
-
ELSE 11
|
|
2385
|
-
END`;
|
|
2386
|
-
};
|
|
2387
|
-
|
|
2388
|
-
const wikiBestMatch = (
|
|
2389
|
-
collectionExpr: string,
|
|
2390
|
-
targetRefExpr: string
|
|
2391
|
-
): string => `
|
|
2392
|
-
SELECT t.id FROM documents t
|
|
2393
|
-
WHERE t.active = 1
|
|
2394
|
-
AND t.collection = ${collectionExpr}
|
|
2395
|
-
AND ${wikiMatch("t", targetRefExpr)}
|
|
2396
|
-
AND ${wikiOrder("t", targetRefExpr)} = (
|
|
2397
|
-
SELECT MIN(${wikiOrder("t2", targetRefExpr)}) FROM documents t2
|
|
2398
|
-
WHERE t2.active = 1
|
|
2399
|
-
AND t2.collection = ${collectionExpr}
|
|
2400
|
-
AND ${wikiMatch("t2", targetRefExpr)}
|
|
2401
|
-
)
|
|
2402
|
-
ORDER BY t.id LIMIT 1
|
|
2403
|
-
`;
|
|
2404
|
-
|
|
2405
|
-
const wikiBestRank = (
|
|
2406
|
-
collectionExpr: string,
|
|
2407
|
-
targetRefExpr: string
|
|
2408
|
-
): string => `
|
|
2409
|
-
SELECT MIN(${wikiOrder("t", targetRefExpr)}) FROM documents t
|
|
2410
|
-
WHERE t.active = 1
|
|
2411
|
-
AND t.collection = ${collectionExpr}
|
|
2412
|
-
AND ${wikiMatch("t", targetRefExpr)}
|
|
2413
|
-
`;
|
|
2414
|
-
|
|
2415
|
-
const wikiBestRankMatchCount = (
|
|
2416
|
-
collectionExpr: string,
|
|
2417
|
-
targetRefExpr: string
|
|
2418
|
-
): string => `
|
|
2419
|
-
SELECT COUNT(*) FROM documents t
|
|
2420
|
-
WHERE t.active = 1
|
|
2421
|
-
AND t.collection = ${collectionExpr}
|
|
2422
|
-
AND ${wikiMatch("t", targetRefExpr)}
|
|
2423
|
-
AND ${wikiOrder("t", targetRefExpr)} = (${wikiBestRank(
|
|
2424
|
-
collectionExpr,
|
|
2425
|
-
targetRefExpr
|
|
2426
|
-
)})
|
|
2427
|
-
`;
|
|
2428
3119
|
interface ResolvedEdgeRow {
|
|
2429
3120
|
source_id: number;
|
|
2430
3121
|
source_docid: string;
|
|
@@ -2464,14 +3155,14 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2464
3155
|
tgt.docid as target_docid,
|
|
2465
3156
|
dl.link_type,
|
|
2466
3157
|
CASE dl.link_type
|
|
2467
|
-
WHEN 'wiki' THEN (${
|
|
3158
|
+
WHEN 'wiki' THEN (${buildWikiBestRankSubquery(
|
|
2468
3159
|
"COALESCE(dl.target_collection, src.collection)",
|
|
2469
3160
|
"dl.target_ref_norm"
|
|
2470
3161
|
)})
|
|
2471
3162
|
WHEN 'markdown' THEN 5
|
|
2472
3163
|
END as match_rank,
|
|
2473
3164
|
CASE dl.link_type
|
|
2474
|
-
WHEN 'wiki' THEN (${
|
|
3165
|
+
WHEN 'wiki' THEN (${buildWikiBestRankMatchCountSubquery(
|
|
2475
3166
|
"COALESCE(dl.target_collection, src.collection)",
|
|
2476
3167
|
"dl.target_ref_norm"
|
|
2477
3168
|
)})
|
|
@@ -2480,7 +3171,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2480
3171
|
FROM documents src
|
|
2481
3172
|
JOIN doc_links dl ON dl.source_doc_id = src.id
|
|
2482
3173
|
JOIN documents tgt ON tgt.id = CASE dl.link_type
|
|
2483
|
-
WHEN 'wiki' THEN (${
|
|
3174
|
+
WHEN 'wiki' THEN (${buildWikiBestMatchSubquery(
|
|
2484
3175
|
"COALESCE(dl.target_collection, src.collection)",
|
|
2485
3176
|
"dl.target_ref_norm"
|
|
2486
3177
|
)})
|
|
@@ -2516,7 +3207,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2516
3207
|
dl.link_type,
|
|
2517
3208
|
CASE dl.link_type
|
|
2518
3209
|
WHEN 'wiki' THEN (
|
|
2519
|
-
${
|
|
3210
|
+
${buildWikiBestMatchSubquery(
|
|
2520
3211
|
"COALESCE(dl.target_collection, src.collection)",
|
|
2521
3212
|
"dl.target_ref_norm"
|
|
2522
3213
|
)}
|
|
@@ -3527,6 +4218,7 @@ interface DbDocumentRow {
|
|
|
3527
4218
|
converter_version: string | null;
|
|
3528
4219
|
language_hint: string | null;
|
|
3529
4220
|
content_type: string | null;
|
|
4221
|
+
content_type_source: string | null;
|
|
3530
4222
|
categories: string | null;
|
|
3531
4223
|
author: string | null;
|
|
3532
4224
|
frontmatter_date: string | null;
|
|
@@ -3542,6 +4234,20 @@ interface DbDocumentRow {
|
|
|
3542
4234
|
updated_at: string;
|
|
3543
4235
|
}
|
|
3544
4236
|
|
|
4237
|
+
interface DbDocEdgeRow {
|
|
4238
|
+
src_doc_id: number;
|
|
4239
|
+
source_docid: string;
|
|
4240
|
+
source_uri: string;
|
|
4241
|
+
source_title: string | null;
|
|
4242
|
+
dst_doc_id: number;
|
|
4243
|
+
target_docid: string;
|
|
4244
|
+
target_uri: string;
|
|
4245
|
+
target_title: string | null;
|
|
4246
|
+
edge_type: string;
|
|
4247
|
+
confidence: DocEdgeConfidence;
|
|
4248
|
+
source: DocEdgeSource;
|
|
4249
|
+
}
|
|
4250
|
+
|
|
3545
4251
|
interface DbChunkRow {
|
|
3546
4252
|
mirror_hash: string;
|
|
3547
4253
|
seq: number;
|
|
@@ -3645,6 +4351,7 @@ function mapDocumentRow(row: DbDocumentRow): DocumentRow {
|
|
|
3645
4351
|
converterVersion: row.converter_version,
|
|
3646
4352
|
languageHint: row.language_hint,
|
|
3647
4353
|
contentType: row.content_type,
|
|
4354
|
+
contentTypeSource: row.content_type_source,
|
|
3648
4355
|
categories,
|
|
3649
4356
|
author: row.author,
|
|
3650
4357
|
frontmatterDate: row.frontmatter_date,
|
|
@@ -3661,6 +4368,23 @@ function mapDocumentRow(row: DbDocumentRow): DocumentRow {
|
|
|
3661
4368
|
};
|
|
3662
4369
|
}
|
|
3663
4370
|
|
|
4371
|
+
function mapDocEdgeRow(row: DbDocEdgeRow): DocEdgeRow {
|
|
4372
|
+
return {
|
|
4373
|
+
sourceDocId: row.src_doc_id,
|
|
4374
|
+
sourceDocid: row.source_docid,
|
|
4375
|
+
sourceUri: row.source_uri,
|
|
4376
|
+
sourceTitle: row.source_title,
|
|
4377
|
+
targetDocId: row.dst_doc_id,
|
|
4378
|
+
targetDocid: row.target_docid,
|
|
4379
|
+
targetUri: row.target_uri,
|
|
4380
|
+
targetTitle: row.target_title,
|
|
4381
|
+
edgeType: row.edge_type,
|
|
4382
|
+
relationType: row.edge_type,
|
|
4383
|
+
confidence: row.confidence,
|
|
4384
|
+
edgeSource: row.source,
|
|
4385
|
+
};
|
|
4386
|
+
}
|
|
4387
|
+
|
|
3664
4388
|
function mapChunkRow(row: DbChunkRow): ChunkRow {
|
|
3665
4389
|
return {
|
|
3666
4390
|
mirrorHash: row.mirror_hash,
|