@equationalapplications/core-llm-wiki 5.3.1 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/dist/{chunk-KIJINNYU.mjs → chunk-QVY5DFJV.mjs} +566 -109
- package/dist/chunk-QVY5DFJV.mjs.map +1 -0
- package/dist/index.d.mts +10 -5
- package/dist/index.d.ts +10 -5
- package/dist/index.js +1191 -154
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +629 -50
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-zf3Jp_RL.d.mts → testing-D0RnZjyW.d.mts} +212 -11
- package/dist/{testing-zf3Jp_RL.d.ts → testing-D0RnZjyW.d.ts} +212 -11
- package/dist/testing.d.mts +2 -1
- package/dist/testing.d.ts +2 -1
- package/dist/testing.js +563 -106
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-KIJINNYU.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var MiniSearch = require('minisearch');
|
|
4
3
|
var coreOkf = require('@equationalapplications/core-okf');
|
|
4
|
+
var MiniSearch = require('minisearch');
|
|
5
5
|
|
|
6
6
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
7
7
|
|
|
@@ -37,7 +37,15 @@ async function setupDatabase(db, prefix) {
|
|
|
37
37
|
embedding_blob BLOB,
|
|
38
38
|
okf_type TEXT,
|
|
39
39
|
ontology_checked_at INTEGER,
|
|
40
|
-
heal_checked_at INTEGER
|
|
40
|
+
heal_checked_at INTEGER,
|
|
41
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
42
|
+
stale_after INTEGER,
|
|
43
|
+
generated_by TEXT,
|
|
44
|
+
last_verified_at INTEGER,
|
|
45
|
+
last_verified_by TEXT,
|
|
46
|
+
okf_sources TEXT,
|
|
47
|
+
okf_verified TEXT,
|
|
48
|
+
okf_usage_window TEXT
|
|
41
49
|
);
|
|
42
50
|
|
|
43
51
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
@@ -73,7 +81,15 @@ async function setupDatabase(db, prefix) {
|
|
|
73
81
|
updated_at INTEGER NOT NULL,
|
|
74
82
|
resolved_at INTEGER,
|
|
75
83
|
deleted_at INTEGER,
|
|
76
|
-
okf_type TEXT
|
|
84
|
+
okf_type TEXT,
|
|
85
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
86
|
+
stale_after INTEGER,
|
|
87
|
+
generated_by TEXT,
|
|
88
|
+
last_verified_at INTEGER,
|
|
89
|
+
last_verified_by TEXT,
|
|
90
|
+
okf_sources TEXT,
|
|
91
|
+
okf_verified TEXT,
|
|
92
|
+
okf_usage_window TEXT
|
|
77
93
|
);
|
|
78
94
|
|
|
79
95
|
CREATE INDEX IF NOT EXISTS ${prefix}tasks_entity_idx ON ${prefix}tasks(entity_id, status);
|
|
@@ -135,6 +151,24 @@ async function setupDatabase(db, prefix) {
|
|
|
135
151
|
CREATE INDEX IF NOT EXISTS ${prefix}outbox_created_at
|
|
136
152
|
ON ${prefix}outbox (created_at);
|
|
137
153
|
`);
|
|
154
|
+
await createOkfV02IndexesIfColumnsExist(db, prefix);
|
|
155
|
+
}
|
|
156
|
+
async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
157
|
+
for (const table of ["entries", "tasks"]) {
|
|
158
|
+
const cols = await db.getAllAsync(
|
|
159
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
160
|
+
);
|
|
161
|
+
const colNames = new Set(cols.map((c) => c.name));
|
|
162
|
+
if (colNames.has("lifecycle_status")) {
|
|
163
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_lifecycle_status_idx ON ${prefix}${table}(lifecycle_status);`);
|
|
164
|
+
}
|
|
165
|
+
if (colNames.has("stale_after")) {
|
|
166
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_stale_after_idx ON ${prefix}${table}(stale_after);`);
|
|
167
|
+
}
|
|
168
|
+
if (colNames.has("last_verified_at")) {
|
|
169
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_last_verified_at_idx ON ${prefix}${table}(last_verified_at);`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
138
172
|
}
|
|
139
173
|
|
|
140
174
|
// src/db/sqliteCodes.ts
|
|
@@ -253,6 +287,25 @@ var PrunePartialFailureError = class extends Error {
|
|
|
253
287
|
}
|
|
254
288
|
};
|
|
255
289
|
var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
|
|
290
|
+
var WikiParseError = class extends Error {
|
|
291
|
+
constructor(message, opts) {
|
|
292
|
+
super(message);
|
|
293
|
+
this.name = "WikiParseError";
|
|
294
|
+
this.tier = opts.tier;
|
|
295
|
+
this.position = opts.position ?? null;
|
|
296
|
+
this.slice = opts.slice ?? "";
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
var WikiIngestEmptyError = class extends Error {
|
|
300
|
+
constructor(params) {
|
|
301
|
+
const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
|
|
302
|
+
super(summary);
|
|
303
|
+
this.name = "WikiIngestEmptyError";
|
|
304
|
+
this.parseFailures = params.parseFailures;
|
|
305
|
+
this.sourceRef = params.sourceRef;
|
|
306
|
+
this.chunks = params.chunks;
|
|
307
|
+
}
|
|
308
|
+
};
|
|
256
309
|
|
|
257
310
|
// src/db/serializedAdapter.ts
|
|
258
311
|
function isDriverError(err) {
|
|
@@ -481,6 +534,39 @@ var MIGRATIONS = [
|
|
|
481
534
|
WHERE rn = 1;`
|
|
482
535
|
);
|
|
483
536
|
}
|
|
537
|
+
},
|
|
538
|
+
{
|
|
539
|
+
version: 10,
|
|
540
|
+
description: "OKF v0.2: add lifecycle_status, stale_after, generated_by, last_verified_*, okf_sources, okf_verified, okf_usage_window to entries and tasks",
|
|
541
|
+
run: async (db, prefix) => {
|
|
542
|
+
for (const table of ["entries", "tasks"]) {
|
|
543
|
+
const cols = await db.getAllAsync(
|
|
544
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
545
|
+
);
|
|
546
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
547
|
+
const adds = [
|
|
548
|
+
["lifecycle_status", `ALTER TABLE ${prefix}${table} ADD COLUMN lifecycle_status TEXT NOT NULL DEFAULT 'stable'`],
|
|
549
|
+
["stale_after", `ALTER TABLE ${prefix}${table} ADD COLUMN stale_after INTEGER`],
|
|
550
|
+
["generated_by", `ALTER TABLE ${prefix}${table} ADD COLUMN generated_by TEXT`],
|
|
551
|
+
["last_verified_at", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_at INTEGER`],
|
|
552
|
+
["last_verified_by", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_by TEXT`],
|
|
553
|
+
["okf_sources", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_sources TEXT`],
|
|
554
|
+
["okf_verified", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_verified TEXT`],
|
|
555
|
+
["okf_usage_window", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_usage_window TEXT`]
|
|
556
|
+
];
|
|
557
|
+
for (const [name, sql] of adds) {
|
|
558
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
await db.execAsync(`
|
|
562
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_lifecycle_status_idx ON ${prefix}entries(lifecycle_status);
|
|
563
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_stale_after_idx ON ${prefix}entries(stale_after);
|
|
564
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_last_verified_at_idx ON ${prefix}entries(last_verified_at);
|
|
565
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_lifecycle_status_idx ON ${prefix}tasks(lifecycle_status);
|
|
566
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_stale_after_idx ON ${prefix}tasks(stale_after);
|
|
567
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
|
|
568
|
+
`);
|
|
569
|
+
}
|
|
484
570
|
}
|
|
485
571
|
];
|
|
486
572
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -507,7 +593,29 @@ var BaseRepository = class {
|
|
|
507
593
|
}
|
|
508
594
|
};
|
|
509
595
|
|
|
510
|
-
// src/repositories/
|
|
596
|
+
// src/repositories/rowMappers.ts
|
|
597
|
+
function parseJsonArray(s, fallback) {
|
|
598
|
+
if (Array.isArray(s)) return s;
|
|
599
|
+
if (typeof s === "string" && s.length > 0) {
|
|
600
|
+
try {
|
|
601
|
+
const p = JSON.parse(s);
|
|
602
|
+
if (Array.isArray(p)) return p;
|
|
603
|
+
} catch {
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
return fallback;
|
|
607
|
+
}
|
|
608
|
+
function parseJsonObject(s, fallback = null) {
|
|
609
|
+
if (s && typeof s === "object" && !Array.isArray(s)) return s;
|
|
610
|
+
if (typeof s === "string" && s.length > 0) {
|
|
611
|
+
try {
|
|
612
|
+
const p = JSON.parse(s);
|
|
613
|
+
if (p && typeof p === "object" && !Array.isArray(p)) return p;
|
|
614
|
+
} catch {
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
return fallback;
|
|
618
|
+
}
|
|
511
619
|
function mapRowToFact(row) {
|
|
512
620
|
const tags = (() => {
|
|
513
621
|
if (Array.isArray(row.tags)) return row.tags;
|
|
@@ -518,6 +626,9 @@ function mapRowToFact(row) {
|
|
|
518
626
|
}
|
|
519
627
|
return [];
|
|
520
628
|
})();
|
|
629
|
+
const okf_verified = parseJsonArray(row.okf_verified, []);
|
|
630
|
+
const now = Date.now();
|
|
631
|
+
const staleAfterRaw = row.stale_after != null ? Number(row.stale_after) : null;
|
|
521
632
|
return {
|
|
522
633
|
id: row.id,
|
|
523
634
|
entity_id: row.entity_id,
|
|
@@ -533,7 +644,19 @@ function mapRowToFact(row) {
|
|
|
533
644
|
last_accessed_at: row.last_accessed_at === null || row.last_accessed_at === void 0 ? null : Number(row.last_accessed_at),
|
|
534
645
|
deleted_at: row.deleted_at != null ? Number(row.deleted_at) : null,
|
|
535
646
|
access_count: Number(row.access_count ?? 0),
|
|
536
|
-
okf_type: row.okf_type ?? null
|
|
647
|
+
okf_type: row.okf_type ?? null,
|
|
648
|
+
// OKF v0.2
|
|
649
|
+
lifecycle_status: row.lifecycle_status ?? "stable",
|
|
650
|
+
stale_after: staleAfterRaw,
|
|
651
|
+
generated_by: row.generated_by ?? null,
|
|
652
|
+
okf_sources: parseJsonArray(row.okf_sources, []),
|
|
653
|
+
okf_verified,
|
|
654
|
+
okf_usage_window: parseJsonObject(row.okf_usage_window),
|
|
655
|
+
last_verified_at: row.last_verified_at != null ? Number(row.last_verified_at) : null,
|
|
656
|
+
last_verified_by: row.last_verified_by ?? null,
|
|
657
|
+
// Spec §2.7 + §5.3: hydrate so read() consumers don't re-call the helpers.
|
|
658
|
+
isStale: coreOkf.isStaleAfter(staleAfterRaw, now),
|
|
659
|
+
trustTier: coreOkf.deriveTrustTier(okf_verified)
|
|
537
660
|
};
|
|
538
661
|
}
|
|
539
662
|
function normalizeEmbeddingBlobValue(blob) {
|
|
@@ -589,6 +712,21 @@ var EntryRepository = class extends BaseRepository {
|
|
|
589
712
|
* Upsert a WikiFact. Nullable fields set to null when fact value is null.
|
|
590
713
|
* Returns { changes, lastInsertRowId }.
|
|
591
714
|
* `tx` is REQUIRED to ensure atomic outbox staging.
|
|
715
|
+
*
|
|
716
|
+
* OKF metadata discipline (spec §2.5 + DAO checklist §8): content writes
|
|
717
|
+
* MUST NOT silently overwrite OKF metadata that a caller didn't supply. The
|
|
718
|
+
* previous CASE WHEN IS NULL guards around `lifecycle_status`, etc. were
|
|
719
|
+
* dead because the bind always supplied a non-null value (notably
|
|
720
|
+
* `fact.lifecycle_status ?? 'stable'`), so excluded.X was never NULL and
|
|
721
|
+
* an UPDATE rewrote an existing `deprecated` row to `stable`. We now omit
|
|
722
|
+
* these columns from the SET clause entirely — SQLite's ON CONFLICT UPDATE
|
|
723
|
+
* leaves untouched columns at their existing row value, which is exactly
|
|
724
|
+
* the preservation semantics we want for content writes. Callers that
|
|
725
|
+
* intentionally want to overwrite OKF metadata should use
|
|
726
|
+
* {@link upsertForImport} (always-replace semantics for that path).
|
|
727
|
+
* New-row INSERTs still bind a value so the column is populated; if the
|
|
728
|
+
* caller supplied nothing, the schema DEFAULT 'stable' applies when the
|
|
729
|
+
* bind is bound as 'stable' explicitly here.
|
|
592
730
|
*/
|
|
593
731
|
async upsert(fact, tx) {
|
|
594
732
|
const executor = this.getExecutor(tx);
|
|
@@ -596,7 +734,10 @@ var EntryRepository = class extends BaseRepository {
|
|
|
596
734
|
const tagsJson = JSON.stringify(fact.tags);
|
|
597
735
|
const embeddingBlob = this.normalizeEmbeddingBlob(fact.embedding_blob);
|
|
598
736
|
const existingRow = await executor.getFirstAsync(
|
|
599
|
-
`SELECT id
|
|
737
|
+
`SELECT id, lifecycle_status, stale_after, generated_by,
|
|
738
|
+
okf_sources, okf_verified, okf_usage_window,
|
|
739
|
+
last_verified_at, last_verified_by
|
|
740
|
+
FROM ${this.prefix}entries WHERE id = ?`,
|
|
600
741
|
[fact.id]
|
|
601
742
|
);
|
|
602
743
|
const operation = fact.deleted_at ? "DELETE" : existingRow ? "UPDATE" : "INSERT";
|
|
@@ -604,8 +745,10 @@ var EntryRepository = class extends BaseRepository {
|
|
|
604
745
|
`INSERT INTO ${this.prefix}entries (
|
|
605
746
|
id, entity_id, title, body, tags, confidence, source_type,
|
|
606
747
|
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
607
|
-
deleted_at, embedding_blob, embedding, okf_type
|
|
608
|
-
|
|
748
|
+
deleted_at, embedding_blob, embedding, okf_type,
|
|
749
|
+
lifecycle_status, stale_after, generated_by, last_verified_at, last_verified_by,
|
|
750
|
+
okf_sources, okf_verified, okf_usage_window
|
|
751
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
609
752
|
ON CONFLICT(id) DO UPDATE SET
|
|
610
753
|
entity_id = excluded.entity_id,
|
|
611
754
|
title = excluded.title,
|
|
@@ -639,15 +782,36 @@ var EntryRepository = class extends BaseRepository {
|
|
|
639
782
|
fact.deleted_at ?? null,
|
|
640
783
|
embeddingBlob ?? null,
|
|
641
784
|
null,
|
|
642
|
-
fact.okf_type ?? null
|
|
785
|
+
fact.okf_type ?? null,
|
|
786
|
+
// OKF v0.2 metadata — bound for INSERT only; ON CONFLICT UPDATE
|
|
787
|
+
// preserves the existing row values (these columns are intentionally
|
|
788
|
+
// absent from the SET clause above).
|
|
789
|
+
fact.lifecycle_status ?? "stable",
|
|
790
|
+
fact.stale_after ?? null,
|
|
791
|
+
fact.generated_by ?? null,
|
|
792
|
+
fact.last_verified_at ?? null,
|
|
793
|
+
fact.last_verified_by ?? null,
|
|
794
|
+
fact.okf_sources ? JSON.stringify(fact.okf_sources) : null,
|
|
795
|
+
fact.okf_verified ? JSON.stringify(fact.okf_verified) : null,
|
|
796
|
+
fact.okf_usage_window ? JSON.stringify(fact.okf_usage_window) : null
|
|
643
797
|
]
|
|
644
798
|
);
|
|
799
|
+
const persistedOkf = existingRow ? {
|
|
800
|
+
lifecycle_status: existingRow.lifecycle_status ?? "stable",
|
|
801
|
+
stale_after: existingRow.stale_after != null ? Number(existingRow.stale_after) : null,
|
|
802
|
+
generated_by: existingRow.generated_by ?? null,
|
|
803
|
+
okf_sources: parseJsonArray(existingRow.okf_sources, []),
|
|
804
|
+
okf_verified: parseJsonArray(existingRow.okf_verified, []),
|
|
805
|
+
okf_usage_window: parseJsonObject(existingRow.okf_usage_window),
|
|
806
|
+
last_verified_at: existingRow.last_verified_at != null ? Number(existingRow.last_verified_at) : null,
|
|
807
|
+
last_verified_by: existingRow.last_verified_by ?? null
|
|
808
|
+
} : null;
|
|
645
809
|
await this.outbox.push({
|
|
646
810
|
entityId: fact.entity_id,
|
|
647
811
|
tableName: "entries",
|
|
648
812
|
recordId: fact.id,
|
|
649
813
|
operation,
|
|
650
|
-
payload: fact
|
|
814
|
+
payload: persistedOkf ? { ...fact, ...persistedOkf } : fact
|
|
651
815
|
}, tx);
|
|
652
816
|
return result;
|
|
653
817
|
}
|
|
@@ -708,8 +872,10 @@ var EntryRepository = class extends BaseRepository {
|
|
|
708
872
|
`INSERT INTO ${this.prefix}entries (
|
|
709
873
|
id, entity_id, title, body, tags, confidence, source_type,
|
|
710
874
|
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
711
|
-
deleted_at, embedding_blob, embedding, okf_type
|
|
712
|
-
|
|
875
|
+
deleted_at, embedding_blob, embedding, okf_type,
|
|
876
|
+
lifecycle_status, stale_after, generated_by, last_verified_at, last_verified_by,
|
|
877
|
+
okf_sources, okf_verified, okf_usage_window
|
|
878
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
713
879
|
ON CONFLICT(id) DO UPDATE SET
|
|
714
880
|
entity_id = excluded.entity_id,
|
|
715
881
|
title = excluded.title,
|
|
@@ -726,7 +892,15 @@ var EntryRepository = class extends BaseRepository {
|
|
|
726
892
|
deleted_at = excluded.deleted_at,
|
|
727
893
|
embedding_blob = excluded.embedding_blob,
|
|
728
894
|
embedding = NULL,
|
|
729
|
-
okf_type = excluded.okf_type
|
|
895
|
+
okf_type = excluded.okf_type,
|
|
896
|
+
lifecycle_status = excluded.lifecycle_status,
|
|
897
|
+
stale_after = excluded.stale_after,
|
|
898
|
+
generated_by = excluded.generated_by,
|
|
899
|
+
last_verified_at = excluded.last_verified_at,
|
|
900
|
+
last_verified_by = excluded.last_verified_by,
|
|
901
|
+
okf_sources = excluded.okf_sources,
|
|
902
|
+
okf_verified = excluded.okf_verified,
|
|
903
|
+
okf_usage_window = excluded.okf_usage_window`,
|
|
730
904
|
[
|
|
731
905
|
fact.id,
|
|
732
906
|
fact.entity_id,
|
|
@@ -744,7 +918,16 @@ var EntryRepository = class extends BaseRepository {
|
|
|
744
918
|
fact.deleted_at ?? null,
|
|
745
919
|
embeddingBlob ?? null,
|
|
746
920
|
null,
|
|
747
|
-
fact.okf_type ?? null
|
|
921
|
+
fact.okf_type ?? null,
|
|
922
|
+
// Same NOT NULL DEFAULT note as in upsert above.
|
|
923
|
+
fact.lifecycle_status ?? "stable",
|
|
924
|
+
fact.stale_after ?? null,
|
|
925
|
+
fact.generated_by ?? null,
|
|
926
|
+
fact.last_verified_at ?? null,
|
|
927
|
+
fact.last_verified_by ?? null,
|
|
928
|
+
fact.okf_sources ? JSON.stringify(fact.okf_sources) : null,
|
|
929
|
+
fact.okf_verified ? JSON.stringify(fact.okf_verified) : null,
|
|
930
|
+
fact.okf_usage_window ? JSON.stringify(fact.okf_usage_window) : null
|
|
748
931
|
]
|
|
749
932
|
);
|
|
750
933
|
return result;
|
|
@@ -1696,9 +1879,10 @@ var SourceRefIndexRepository = class extends BaseRepository {
|
|
|
1696
1879
|
return row?.source_ref ?? null;
|
|
1697
1880
|
}
|
|
1698
1881
|
};
|
|
1699
|
-
|
|
1700
|
-
// src/repositories/TaskRepository.ts
|
|
1701
1882
|
function mapRowToTask(row) {
|
|
1883
|
+
const okf_verified = parseJsonArray(row.okf_verified, []);
|
|
1884
|
+
const now = Date.now();
|
|
1885
|
+
const staleAfterRaw = row.stale_after != null ? Number(row.stale_after) : null;
|
|
1702
1886
|
return {
|
|
1703
1887
|
id: row.id,
|
|
1704
1888
|
entity_id: row.entity_id,
|
|
@@ -1709,7 +1893,19 @@ function mapRowToTask(row) {
|
|
|
1709
1893
|
updated_at: Number(row.updated_at),
|
|
1710
1894
|
resolved_at: row.resolved_at != null ? Number(row.resolved_at) : null,
|
|
1711
1895
|
deleted_at: row.deleted_at != null ? Number(row.deleted_at) : null,
|
|
1712
|
-
okf_type: row.okf_type ?? null
|
|
1896
|
+
okf_type: row.okf_type ?? null,
|
|
1897
|
+
// OKF v0.2
|
|
1898
|
+
lifecycle_status: row.lifecycle_status ?? "stable",
|
|
1899
|
+
stale_after: staleAfterRaw,
|
|
1900
|
+
generated_by: row.generated_by ?? null,
|
|
1901
|
+
okf_sources: parseJsonArray(row.okf_sources, []),
|
|
1902
|
+
okf_verified,
|
|
1903
|
+
okf_usage_window: parseJsonObject(row.okf_usage_window),
|
|
1904
|
+
last_verified_at: row.last_verified_at != null ? Number(row.last_verified_at) : null,
|
|
1905
|
+
last_verified_by: row.last_verified_by ?? null,
|
|
1906
|
+
// Spec §2.7 + §5.3: hydrate so read() consumers don't re-call the helpers.
|
|
1907
|
+
isStale: coreOkf.isStaleAfter(staleAfterRaw, now),
|
|
1908
|
+
trustTier: coreOkf.deriveTrustTier(okf_verified)
|
|
1713
1909
|
};
|
|
1714
1910
|
}
|
|
1715
1911
|
var TaskRepository = class extends BaseRepository {
|
|
@@ -1759,20 +1955,31 @@ var TaskRepository = class extends BaseRepository {
|
|
|
1759
1955
|
* Uses ON CONFLICT(id) DO UPDATE (not INSERT OR REPLACE).
|
|
1760
1956
|
* Stages an outbox entry in the same transaction.
|
|
1761
1957
|
* `tx` is REQUIRED.
|
|
1958
|
+
*
|
|
1959
|
+
* OKF metadata (lifecycle_status, sources, etc.) is intentionally excluded
|
|
1960
|
+
* from the SET clause so content writes preserve any existing row values.
|
|
1961
|
+
* See {@link EntryRepository.upsert} for the rationale — the same applies
|
|
1962
|
+
* symmetrically to tasks (spec §2.5). Callers that need to overwrite OKF
|
|
1963
|
+
* metadata should use {@link upsertForImport}.
|
|
1762
1964
|
*/
|
|
1763
1965
|
async upsert(task, tx, updatedAt) {
|
|
1764
1966
|
const executor = this.getExecutor(tx);
|
|
1765
1967
|
const now = Number.isFinite(updatedAt) ? updatedAt : Date.now();
|
|
1766
1968
|
const existingRow = await executor.getFirstAsync(
|
|
1767
|
-
`SELECT id
|
|
1969
|
+
`SELECT id, lifecycle_status, stale_after, generated_by,
|
|
1970
|
+
okf_sources, okf_verified, okf_usage_window,
|
|
1971
|
+
last_verified_at, last_verified_by
|
|
1972
|
+
FROM ${this.prefix}tasks WHERE id = ?`,
|
|
1768
1973
|
[task.id]
|
|
1769
1974
|
);
|
|
1770
1975
|
const operation = task.deleted_at != null ? "DELETE" : existingRow ? "UPDATE" : "INSERT";
|
|
1771
1976
|
await executor.runAsync(
|
|
1772
1977
|
`INSERT INTO ${this.prefix}tasks (
|
|
1773
1978
|
id, entity_id, description, status, priority,
|
|
1774
|
-
created_at, updated_at, resolved_at, deleted_at
|
|
1775
|
-
|
|
1979
|
+
created_at, updated_at, resolved_at, deleted_at,
|
|
1980
|
+
lifecycle_status, stale_after, generated_by, last_verified_at, last_verified_by,
|
|
1981
|
+
okf_sources, okf_verified, okf_usage_window
|
|
1982
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1776
1983
|
ON CONFLICT(id) DO UPDATE SET
|
|
1777
1984
|
entity_id = excluded.entity_id,
|
|
1778
1985
|
description = excluded.description,
|
|
@@ -1791,16 +1998,37 @@ var TaskRepository = class extends BaseRepository {
|
|
|
1791
1998
|
now,
|
|
1792
1999
|
// updated_at set by repo or import override
|
|
1793
2000
|
task.resolved_at ?? null,
|
|
1794
|
-
task.deleted_at ?? null
|
|
2001
|
+
task.deleted_at ?? null,
|
|
2002
|
+
// OKF v0.2 metadata — bound for INSERT only; ON CONFLICT UPDATE
|
|
2003
|
+
// preserves the existing row values (these columns are intentionally
|
|
2004
|
+
// absent from the SET clause above).
|
|
2005
|
+
task.lifecycle_status ?? "stable",
|
|
2006
|
+
task.stale_after ?? null,
|
|
2007
|
+
task.generated_by ?? null,
|
|
2008
|
+
task.last_verified_at ?? null,
|
|
2009
|
+
task.last_verified_by ?? null,
|
|
2010
|
+
task.okf_sources ? JSON.stringify(task.okf_sources) : null,
|
|
2011
|
+
task.okf_verified ? JSON.stringify(task.okf_verified) : null,
|
|
2012
|
+
task.okf_usage_window ? JSON.stringify(task.okf_usage_window) : null
|
|
1795
2013
|
]
|
|
1796
2014
|
);
|
|
2015
|
+
const persistedOkf = existingRow ? {
|
|
2016
|
+
lifecycle_status: existingRow.lifecycle_status ?? "stable",
|
|
2017
|
+
stale_after: existingRow.stale_after != null ? Number(existingRow.stale_after) : null,
|
|
2018
|
+
generated_by: existingRow.generated_by ?? null,
|
|
2019
|
+
okf_sources: parseJsonArray(existingRow.okf_sources, []),
|
|
2020
|
+
okf_verified: parseJsonArray(existingRow.okf_verified, []),
|
|
2021
|
+
okf_usage_window: parseJsonObject(existingRow.okf_usage_window),
|
|
2022
|
+
last_verified_at: existingRow.last_verified_at != null ? Number(existingRow.last_verified_at) : null,
|
|
2023
|
+
last_verified_by: existingRow.last_verified_by ?? null
|
|
2024
|
+
} : null;
|
|
1797
2025
|
await this.outbox.push(
|
|
1798
2026
|
{
|
|
1799
2027
|
entityId: task.entity_id,
|
|
1800
2028
|
tableName: "tasks",
|
|
1801
2029
|
recordId: task.id,
|
|
1802
2030
|
operation,
|
|
1803
|
-
payload: task
|
|
2031
|
+
payload: persistedOkf ? { ...task, ...persistedOkf } : task
|
|
1804
2032
|
},
|
|
1805
2033
|
tx
|
|
1806
2034
|
);
|
|
@@ -1811,8 +2039,10 @@ var TaskRepository = class extends BaseRepository {
|
|
|
1811
2039
|
await executor.runAsync(
|
|
1812
2040
|
`INSERT INTO ${this.prefix}tasks (
|
|
1813
2041
|
id, entity_id, description, status, priority,
|
|
1814
|
-
created_at, updated_at, resolved_at, deleted_at, okf_type
|
|
1815
|
-
|
|
2042
|
+
created_at, updated_at, resolved_at, deleted_at, okf_type,
|
|
2043
|
+
lifecycle_status, stale_after, generated_by, last_verified_at, last_verified_by,
|
|
2044
|
+
okf_sources, okf_verified, okf_usage_window
|
|
2045
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1816
2046
|
ON CONFLICT(id) DO UPDATE SET
|
|
1817
2047
|
entity_id = excluded.entity_id,
|
|
1818
2048
|
description = excluded.description,
|
|
@@ -1821,7 +2051,15 @@ var TaskRepository = class extends BaseRepository {
|
|
|
1821
2051
|
updated_at = excluded.updated_at,
|
|
1822
2052
|
resolved_at = excluded.resolved_at,
|
|
1823
2053
|
deleted_at = excluded.deleted_at,
|
|
1824
|
-
okf_type = excluded.okf_type
|
|
2054
|
+
okf_type = excluded.okf_type,
|
|
2055
|
+
lifecycle_status = excluded.lifecycle_status,
|
|
2056
|
+
stale_after = excluded.stale_after,
|
|
2057
|
+
generated_by = excluded.generated_by,
|
|
2058
|
+
last_verified_at = excluded.last_verified_at,
|
|
2059
|
+
last_verified_by = excluded.last_verified_by,
|
|
2060
|
+
okf_sources = excluded.okf_sources,
|
|
2061
|
+
okf_verified = excluded.okf_verified,
|
|
2062
|
+
okf_usage_window = excluded.okf_usage_window`,
|
|
1825
2063
|
[
|
|
1826
2064
|
task.id,
|
|
1827
2065
|
task.entity_id,
|
|
@@ -1832,7 +2070,15 @@ var TaskRepository = class extends BaseRepository {
|
|
|
1832
2070
|
now,
|
|
1833
2071
|
task.resolved_at ?? null,
|
|
1834
2072
|
task.deleted_at ?? null,
|
|
1835
|
-
task.okf_type ?? null
|
|
2073
|
+
task.okf_type ?? null,
|
|
2074
|
+
task.lifecycle_status ?? "stable",
|
|
2075
|
+
task.stale_after ?? null,
|
|
2076
|
+
task.generated_by ?? null,
|
|
2077
|
+
task.last_verified_at ?? null,
|
|
2078
|
+
task.last_verified_by ?? null,
|
|
2079
|
+
task.okf_sources ? JSON.stringify(task.okf_sources) : null,
|
|
2080
|
+
task.okf_verified ? JSON.stringify(task.okf_verified) : null,
|
|
2081
|
+
task.okf_usage_window ? JSON.stringify(task.okf_usage_window) : null
|
|
1836
2082
|
]
|
|
1837
2083
|
);
|
|
1838
2084
|
}
|
|
@@ -3190,24 +3436,67 @@ var JobManager = class {
|
|
|
3190
3436
|
};
|
|
3191
3437
|
|
|
3192
3438
|
// src/utils/pure.ts
|
|
3439
|
+
var MAX_REPAIR_CANDIDATES = 5;
|
|
3440
|
+
function throwNoJsonFound(text, start) {
|
|
3441
|
+
throw new WikiParseError(
|
|
3442
|
+
"No JSON object/array found in LLM response",
|
|
3443
|
+
{ tier: "strict", position: start, slice: text }
|
|
3444
|
+
);
|
|
3445
|
+
}
|
|
3193
3446
|
function parseJsonResponse(text) {
|
|
3194
3447
|
const firstBrace = text.indexOf("{");
|
|
3195
3448
|
const firstBracket = text.indexOf("[");
|
|
3196
3449
|
let start;
|
|
3197
3450
|
let openChar;
|
|
3198
|
-
let closeChar;
|
|
3199
3451
|
const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
|
|
3200
3452
|
if (useBrace) {
|
|
3201
3453
|
start = firstBrace;
|
|
3202
3454
|
openChar = "{";
|
|
3203
|
-
closeChar = "}";
|
|
3204
3455
|
} else if (firstBracket !== -1) {
|
|
3205
3456
|
start = firstBracket;
|
|
3206
3457
|
openChar = "[";
|
|
3207
|
-
closeChar = "]";
|
|
3208
3458
|
} else {
|
|
3209
|
-
|
|
3459
|
+
throwNoJsonFound(text, null);
|
|
3460
|
+
}
|
|
3461
|
+
const slice = scanJsonSlice(text, start, openChar);
|
|
3462
|
+
if (slice !== null) {
|
|
3463
|
+
try {
|
|
3464
|
+
return JSON.parse(slice);
|
|
3465
|
+
} catch {
|
|
3466
|
+
}
|
|
3467
|
+
}
|
|
3468
|
+
const repairOdd = containerAwareRepair(text, start, false);
|
|
3469
|
+
if (repairOdd.success !== null && !repairOdd.ambiguous) {
|
|
3470
|
+
return repairOdd.success;
|
|
3471
|
+
}
|
|
3472
|
+
const repairEven = containerAwareRepair(text, start, true);
|
|
3473
|
+
if (repairOdd.success !== null && repairEven.success !== null) {
|
|
3474
|
+
const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
|
|
3475
|
+
return pick.success;
|
|
3476
|
+
}
|
|
3477
|
+
if (repairOdd.success !== null) {
|
|
3478
|
+
return repairOdd.success;
|
|
3479
|
+
}
|
|
3480
|
+
if (repairEven.success !== null) {
|
|
3481
|
+
return repairEven.success;
|
|
3210
3482
|
}
|
|
3483
|
+
const repair = repairEven.failed !== null ? repairEven : repairOdd;
|
|
3484
|
+
if (repair.failed !== null) {
|
|
3485
|
+
throw new WikiParseError(
|
|
3486
|
+
`Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
|
|
3487
|
+
{ tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
|
|
3488
|
+
);
|
|
3489
|
+
}
|
|
3490
|
+
if (slice === null) {
|
|
3491
|
+
throwNoJsonFound(text, start);
|
|
3492
|
+
}
|
|
3493
|
+
throw new WikiParseError(
|
|
3494
|
+
"No parsable JSON candidate found",
|
|
3495
|
+
{ tier: "all", position: start, slice: text }
|
|
3496
|
+
);
|
|
3497
|
+
}
|
|
3498
|
+
function scanJsonSlice(text, start, openChar) {
|
|
3499
|
+
const closeChar = openChar === "{" ? "}" : "]";
|
|
3211
3500
|
let depth = 0;
|
|
3212
3501
|
let inString = false;
|
|
3213
3502
|
let escape = false;
|
|
@@ -3239,8 +3528,193 @@ function parseJsonResponse(text) {
|
|
|
3239
3528
|
}
|
|
3240
3529
|
}
|
|
3241
3530
|
}
|
|
3242
|
-
if (end === -1)
|
|
3243
|
-
return
|
|
3531
|
+
if (end === -1) return null;
|
|
3532
|
+
return safeSlice(text, start, end + 1);
|
|
3533
|
+
}
|
|
3534
|
+
function containerAwareRepair(text, start, closeOnEvenParity) {
|
|
3535
|
+
const stack = [];
|
|
3536
|
+
let inString = false;
|
|
3537
|
+
let stringRole = null;
|
|
3538
|
+
let escape = false;
|
|
3539
|
+
let i = start;
|
|
3540
|
+
let out = "";
|
|
3541
|
+
let bareQuoteCount = 0;
|
|
3542
|
+
let bestFailed = null;
|
|
3543
|
+
let escapes = 0;
|
|
3544
|
+
let ambiguous = false;
|
|
3545
|
+
let bestSuccess = null;
|
|
3546
|
+
let attempts = 0;
|
|
3547
|
+
function tryEmitCandidate(candidate) {
|
|
3548
|
+
attempts++;
|
|
3549
|
+
try {
|
|
3550
|
+
bestSuccess = JSON.parse(candidate);
|
|
3551
|
+
return true;
|
|
3552
|
+
} catch (err) {
|
|
3553
|
+
if (bestFailed === null) {
|
|
3554
|
+
bestFailed = {
|
|
3555
|
+
candidate,
|
|
3556
|
+
position: extractParsePosition(err),
|
|
3557
|
+
message: safeErrorToString(err)
|
|
3558
|
+
};
|
|
3559
|
+
}
|
|
3560
|
+
return false;
|
|
3561
|
+
}
|
|
3562
|
+
}
|
|
3563
|
+
while (i < text.length) {
|
|
3564
|
+
const ch = text[i];
|
|
3565
|
+
if (stack.length === 0 && !inString) {
|
|
3566
|
+
if (ch === "{" || ch === "[") {
|
|
3567
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
3568
|
+
out += ch;
|
|
3569
|
+
}
|
|
3570
|
+
i++;
|
|
3571
|
+
continue;
|
|
3572
|
+
}
|
|
3573
|
+
if (escape) {
|
|
3574
|
+
out += ch;
|
|
3575
|
+
escape = false;
|
|
3576
|
+
i++;
|
|
3577
|
+
continue;
|
|
3578
|
+
}
|
|
3579
|
+
if (inString && ch === "\\") {
|
|
3580
|
+
out += ch;
|
|
3581
|
+
escape = true;
|
|
3582
|
+
i++;
|
|
3583
|
+
continue;
|
|
3584
|
+
}
|
|
3585
|
+
if (inString) {
|
|
3586
|
+
const code = ch.charCodeAt(0);
|
|
3587
|
+
if (code < 32) {
|
|
3588
|
+
out += controlCharEscape(code);
|
|
3589
|
+
i++;
|
|
3590
|
+
continue;
|
|
3591
|
+
}
|
|
3592
|
+
}
|
|
3593
|
+
if (ch === '"') {
|
|
3594
|
+
let j = i + 1;
|
|
3595
|
+
while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
|
|
3596
|
+
j++;
|
|
3597
|
+
}
|
|
3598
|
+
const next = j < text.length ? text[j] : "";
|
|
3599
|
+
if (!inString) {
|
|
3600
|
+
if (next === "}" || next === "]") {
|
|
3601
|
+
if (j > i + 1) out += text.slice(i + 1, j);
|
|
3602
|
+
out += next;
|
|
3603
|
+
i = j + 1;
|
|
3604
|
+
stack.pop();
|
|
3605
|
+
if (stack.length === 0) {
|
|
3606
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
3607
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
3608
|
+
out = "";
|
|
3609
|
+
}
|
|
3610
|
+
continue;
|
|
3611
|
+
}
|
|
3612
|
+
const top = stack[stack.length - 1];
|
|
3613
|
+
stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
|
|
3614
|
+
out += ch;
|
|
3615
|
+
inString = true;
|
|
3616
|
+
i++;
|
|
3617
|
+
continue;
|
|
3618
|
+
}
|
|
3619
|
+
const commaOnly = next === ",";
|
|
3620
|
+
const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
|
|
3621
|
+
const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
|
|
3622
|
+
if (commaOnly && bareQuoteCount > 0) ambiguous = true;
|
|
3623
|
+
const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
|
|
3624
|
+
const isClosing = isKeyClose || isValueClose || shouldCommaClose;
|
|
3625
|
+
if (isClosing) {
|
|
3626
|
+
out += ch;
|
|
3627
|
+
inString = false;
|
|
3628
|
+
const topFrame = stack[stack.length - 1];
|
|
3629
|
+
if (topFrame && topFrame.container === "object") {
|
|
3630
|
+
topFrame.expectKey = stringRole === "value";
|
|
3631
|
+
}
|
|
3632
|
+
stringRole = null;
|
|
3633
|
+
bareQuoteCount = 0;
|
|
3634
|
+
i++;
|
|
3635
|
+
continue;
|
|
3636
|
+
}
|
|
3637
|
+
out += "\\" + ch;
|
|
3638
|
+
bareQuoteCount++;
|
|
3639
|
+
escapes++;
|
|
3640
|
+
i++;
|
|
3641
|
+
continue;
|
|
3642
|
+
}
|
|
3643
|
+
if (!inString && (ch === "{" || ch === "[")) {
|
|
3644
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
3645
|
+
out += ch;
|
|
3646
|
+
i++;
|
|
3647
|
+
continue;
|
|
3648
|
+
}
|
|
3649
|
+
if (!inString && ch === ",") {
|
|
3650
|
+
const topFrame = stack[stack.length - 1];
|
|
3651
|
+
if (topFrame && topFrame.container === "object") {
|
|
3652
|
+
topFrame.expectKey = true;
|
|
3653
|
+
}
|
|
3654
|
+
out += ch;
|
|
3655
|
+
i++;
|
|
3656
|
+
continue;
|
|
3657
|
+
}
|
|
3658
|
+
if (!inString && (ch === "}" || ch === "]")) {
|
|
3659
|
+
out += ch;
|
|
3660
|
+
stack.pop();
|
|
3661
|
+
if (stack.length === 0) {
|
|
3662
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
3663
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
3664
|
+
out = "";
|
|
3665
|
+
}
|
|
3666
|
+
i++;
|
|
3667
|
+
continue;
|
|
3668
|
+
}
|
|
3669
|
+
out += ch;
|
|
3670
|
+
i++;
|
|
3671
|
+
}
|
|
3672
|
+
return { success: null, failed: bestFailed, escapes, ambiguous };
|
|
3673
|
+
}
|
|
3674
|
+
function controlCharEscape(code) {
|
|
3675
|
+
switch (code) {
|
|
3676
|
+
case 10:
|
|
3677
|
+
return "\\n";
|
|
3678
|
+
case 9:
|
|
3679
|
+
return "\\t";
|
|
3680
|
+
case 13:
|
|
3681
|
+
return "\\r";
|
|
3682
|
+
case 8:
|
|
3683
|
+
return "\\b";
|
|
3684
|
+
case 12:
|
|
3685
|
+
return "\\f";
|
|
3686
|
+
default:
|
|
3687
|
+
return "\\u" + code.toString(16).padStart(4, "0");
|
|
3688
|
+
}
|
|
3689
|
+
}
|
|
3690
|
+
function extractParsePosition(err) {
|
|
3691
|
+
const match = /position\s+(\d+)/.exec(safeErrorToString(err));
|
|
3692
|
+
return match ? Number(match[1]) : null;
|
|
3693
|
+
}
|
|
3694
|
+
function safeErrorToString(e) {
|
|
3695
|
+
if (e instanceof Error) {
|
|
3696
|
+
const msg = readErrorField(e, "message");
|
|
3697
|
+
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
3698
|
+
const name = readErrorField(e, "name");
|
|
3699
|
+
if (typeof name === "string" && name.length > 0) return name;
|
|
3700
|
+
return "[Error]";
|
|
3701
|
+
}
|
|
3702
|
+
try {
|
|
3703
|
+
return String(e);
|
|
3704
|
+
} catch {
|
|
3705
|
+
try {
|
|
3706
|
+
return Object.prototype.toString.call(e);
|
|
3707
|
+
} catch {
|
|
3708
|
+
return "[unstringifiable error]";
|
|
3709
|
+
}
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3712
|
+
function readErrorField(e, key) {
|
|
3713
|
+
try {
|
|
3714
|
+
return e[key];
|
|
3715
|
+
} catch {
|
|
3716
|
+
return void 0;
|
|
3717
|
+
}
|
|
3244
3718
|
}
|
|
3245
3719
|
function sanitizeRankerError(err, sanitizeRankerErrors) {
|
|
3246
3720
|
if (sanitizeRankerErrors === false) {
|
|
@@ -3425,7 +3899,7 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
3425
3899
|
{
|
|
3426
3900
|
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
3427
3901
|
}
|
|
3428
|
-
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
3902
|
+
Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
|
|
3429
3903
|
var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
|
|
3430
3904
|
Return ONLY a valid JSON object matching this schema:
|
|
3431
3905
|
{
|
|
@@ -3434,7 +3908,7 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
3434
3908
|
]
|
|
3435
3909
|
}
|
|
3436
3910
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
3437
|
-
Do not return markdown, just raw JSON.`;
|
|
3911
|
+
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
3438
3912
|
|
|
3439
3913
|
// src/services/PromptService.ts
|
|
3440
3914
|
var PromptService = class {
|
|
@@ -3547,6 +4021,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
|
3547
4021
|
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
3548
4022
|
|
|
3549
4023
|
// src/services/IngestionService.ts
|
|
4024
|
+
function zeroChunkResult(duplicateOf) {
|
|
4025
|
+
const result = {
|
|
4026
|
+
truncated: false,
|
|
4027
|
+
chunks: 0,
|
|
4028
|
+
ingestedChunks: 0,
|
|
4029
|
+
failedChunks: 0
|
|
4030
|
+
};
|
|
4031
|
+
if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
|
|
4032
|
+
return result;
|
|
4033
|
+
}
|
|
3550
4034
|
var IngestionService = class {
|
|
3551
4035
|
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3552
4036
|
this.db = db;
|
|
@@ -3587,113 +4071,110 @@ var IngestionService = class {
|
|
|
3587
4071
|
if (onDuplicateHash === "throw") {
|
|
3588
4072
|
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
3589
4073
|
}
|
|
3590
|
-
return
|
|
4074
|
+
return zeroChunkResult(canonical);
|
|
3591
4075
|
}
|
|
3592
4076
|
}
|
|
3593
4077
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
3594
|
-
if (chunks.length === 0) return
|
|
4078
|
+
if (chunks.length === 0) return zeroChunkResult();
|
|
4079
|
+
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3595
4080
|
const chunkResults = await withConcurrency(
|
|
3596
|
-
chunks.map((chunk) => async () => {
|
|
3597
|
-
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
4081
|
+
chunks.map((chunk, chunkIndex) => async () => {
|
|
3598
4082
|
const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
|
|
3599
4083
|
chunk,
|
|
3600
4084
|
params.promptOverride,
|
|
3601
4085
|
ontologyContext
|
|
3602
4086
|
);
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
|
|
4087
|
+
try {
|
|
4088
|
+
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
4089
|
+
const result2 = parseJsonResponse(responseText);
|
|
4090
|
+
return {
|
|
4091
|
+
status: "ok",
|
|
4092
|
+
facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
|
|
4093
|
+
ontology_updates: result2.ontology_updates
|
|
4094
|
+
};
|
|
4095
|
+
} catch (e) {
|
|
4096
|
+
const failure = e instanceof WikiParseError ? {
|
|
4097
|
+
chunkIndex,
|
|
4098
|
+
sourceRef,
|
|
4099
|
+
source: "parse",
|
|
4100
|
+
tier: e.tier,
|
|
4101
|
+
position: e.position,
|
|
4102
|
+
message: e.message
|
|
4103
|
+
} : {
|
|
4104
|
+
chunkIndex,
|
|
4105
|
+
sourceRef,
|
|
4106
|
+
source: "llm",
|
|
4107
|
+
// `safeErrorToString` is a non-throwing coercion: an LLM
|
|
4108
|
+
// provider may reject with a non-Error value, and even
|
|
4109
|
+
// `String(e)` itself can throw when `e.toString()` throws.
|
|
4110
|
+
// Letting such an exception escape this catch would cause
|
|
4111
|
+
// `withConcurrency` to reject, discarding every sibling
|
|
4112
|
+
// chunk's results — the exact failure mode this per-chunk
|
|
4113
|
+
// try/catch exists to prevent. Keeping the value in the
|
|
4114
|
+
// `ChunkFailure` preserves the typed diagnostic for
|
|
4115
|
+
// callers (see parseFailures) without unwinding the loop.
|
|
4116
|
+
position: null,
|
|
4117
|
+
message: safeErrorToString(e)
|
|
4118
|
+
};
|
|
4119
|
+
const total = chunks.length;
|
|
4120
|
+
const tags = [];
|
|
4121
|
+
if (failure.tier) tags.push(`tier=${failure.tier}`);
|
|
4122
|
+
if (failure.position !== null) tags.push(`position=${failure.position}`);
|
|
4123
|
+
const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
|
|
4124
|
+
console.warn(
|
|
4125
|
+
`[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
|
|
4126
|
+
);
|
|
4127
|
+
return { status: "failed", error: failure };
|
|
4128
|
+
}
|
|
3609
4129
|
}),
|
|
3610
4130
|
chunkConcurrency
|
|
3611
4131
|
);
|
|
4132
|
+
let ingestedChunks = 0;
|
|
4133
|
+
let failedChunks = 0;
|
|
4134
|
+
const failures = [];
|
|
3612
4135
|
const seen = /* @__PURE__ */ new Set();
|
|
3613
4136
|
const orderedChunkFacts = [];
|
|
3614
|
-
for (const
|
|
4137
|
+
for (const slot of chunkResults) {
|
|
4138
|
+
if (slot.status === "failed") {
|
|
4139
|
+
failedChunks++;
|
|
4140
|
+
failures.push(slot.error);
|
|
4141
|
+
continue;
|
|
4142
|
+
}
|
|
4143
|
+
ingestedChunks++;
|
|
3615
4144
|
const dedupedFacts = [];
|
|
3616
|
-
for (const fact of
|
|
4145
|
+
for (const fact of slot.facts) {
|
|
3617
4146
|
const normalizedTitle = normalizeTitleKey(fact.title);
|
|
3618
4147
|
if (!seen.has(normalizedTitle)) {
|
|
3619
4148
|
seen.add(normalizedTitle);
|
|
3620
4149
|
dedupedFacts.push(fact);
|
|
3621
4150
|
}
|
|
3622
4151
|
}
|
|
3623
|
-
orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates:
|
|
4152
|
+
orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
|
|
4153
|
+
}
|
|
4154
|
+
if (failedChunks === chunks.length) {
|
|
4155
|
+
throw new WikiIngestEmptyError({
|
|
4156
|
+
parseFailures: failures,
|
|
4157
|
+
sourceRef,
|
|
4158
|
+
chunks: chunks.length
|
|
4159
|
+
});
|
|
3624
4160
|
}
|
|
3625
|
-
const now = Date.now();
|
|
3626
4161
|
const insertedFacts = [];
|
|
3627
4162
|
const deletedSourceFactIds = [];
|
|
3628
4163
|
try {
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
3644
|
-
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
3645
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
3646
|
-
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
3647
|
-
mode = ontologyState.mode;
|
|
3648
|
-
}
|
|
3649
|
-
for (const fact of facts) {
|
|
3650
|
-
const ontologyFact = fact;
|
|
3651
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
3652
|
-
const id = generateId("fact_");
|
|
3653
|
-
hostNodes.push({
|
|
3654
|
-
id,
|
|
3655
|
-
type: ontologyFact.okf_type ?? "",
|
|
3656
|
-
title: fact.title,
|
|
3657
|
-
body: fact.body,
|
|
3658
|
-
// Forward the LLM-extracted tags/confidence through the
|
|
3659
|
-
// extracted-shape upsertGraphCore path so the entries row
|
|
3660
|
-
// stores them (search filterability, heal-candidate
|
|
3661
|
-
// selection, runReembed embedding-text signal). The public
|
|
3662
|
-
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
3663
|
-
// — host-supplied deterministic nodes default to [] and
|
|
3664
|
-
// 'certain' as documented.
|
|
3665
|
-
tags: fact.tags,
|
|
3666
|
-
confidence: fact.confidence
|
|
3667
|
-
});
|
|
3668
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3669
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3670
|
-
if (normalized.edges.length > 0) {
|
|
3671
|
-
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3672
|
-
}
|
|
3673
|
-
}
|
|
3674
|
-
}
|
|
3675
|
-
const hostEdges = [];
|
|
3676
|
-
for (const req of rawEdgeRequests) {
|
|
3677
|
-
const resolved = this.ontologyService?.resolveEdges(
|
|
3678
|
-
entityId,
|
|
3679
|
-
req.sourceId,
|
|
3680
|
-
req.sourceType,
|
|
3681
|
-
req.edges,
|
|
3682
|
-
manifest,
|
|
3683
|
-
titleIndex,
|
|
3684
|
-
now
|
|
3685
|
-
) ?? [];
|
|
3686
|
-
for (const e of resolved) {
|
|
3687
|
-
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
3688
|
-
}
|
|
3689
|
-
}
|
|
3690
|
-
await this.upsertGraphCore(
|
|
3691
|
-
entityId,
|
|
3692
|
-
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
3693
|
-
tx,
|
|
3694
|
-
{ strict: false }
|
|
3695
|
-
);
|
|
3696
|
-
});
|
|
4164
|
+
if (failedChunks === 0) {
|
|
4165
|
+
const fullResult = await this.db.withTransactionAsync(async (tx) => {
|
|
4166
|
+
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
|
|
4167
|
+
});
|
|
4168
|
+
deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
|
|
4169
|
+
insertedFacts.push(...fullResult.insertedFacts);
|
|
4170
|
+
} else {
|
|
4171
|
+
const partialResult = await this.db.withTransactionAsync(async (tx) => {
|
|
4172
|
+
const flat = [];
|
|
4173
|
+
for (const slot of orderedChunkFacts) flat.push(...slot.facts);
|
|
4174
|
+
return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
|
|
4175
|
+
});
|
|
4176
|
+
insertedFacts.push(...partialResult.insertedDescriptors);
|
|
4177
|
+
}
|
|
3697
4178
|
} catch (err) {
|
|
3698
4179
|
const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
|
|
3699
4180
|
if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
|
|
@@ -3702,22 +4183,31 @@ var IngestionService = class {
|
|
|
3702
4183
|
if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
|
|
3703
4184
|
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
3704
4185
|
}
|
|
3705
|
-
return
|
|
4186
|
+
return zeroChunkResult(canonical);
|
|
3706
4187
|
}
|
|
3707
4188
|
await this.searchService.sync(entityId);
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
4189
|
+
if (failedChunks === 0) {
|
|
4190
|
+
const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
|
|
4191
|
+
for (const factId of uniqueDeletedSourceFactIds) {
|
|
4192
|
+
try {
|
|
4193
|
+
await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
|
|
4194
|
+
} catch (hookErr) {
|
|
4195
|
+
console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
|
|
4196
|
+
}
|
|
3714
4197
|
}
|
|
3715
4198
|
}
|
|
3716
4199
|
for (const fact of insertedFacts) {
|
|
3717
4200
|
await this.embeddingService.embedFact(fact);
|
|
3718
4201
|
}
|
|
3719
4202
|
this.searchService.evictCache(entityId);
|
|
3720
|
-
|
|
4203
|
+
const result = {
|
|
4204
|
+
truncated,
|
|
4205
|
+
chunks: chunks.length,
|
|
4206
|
+
ingestedChunks,
|
|
4207
|
+
failedChunks
|
|
4208
|
+
};
|
|
4209
|
+
if (failedChunks > 0) result.parseFailures = failures;
|
|
4210
|
+
return result;
|
|
3721
4211
|
} finally {
|
|
3722
4212
|
releaseIngestLocks();
|
|
3723
4213
|
}
|
|
@@ -3842,6 +4332,156 @@ var IngestionService = class {
|
|
|
3842
4332
|
superseded: deletedFactIds.length + deletedEdgeCount
|
|
3843
4333
|
};
|
|
3844
4334
|
}
|
|
4335
|
+
/**
|
|
4336
|
+
* Full supersession + ownership path: identical to the pre-issue-#92 behavior
|
|
4337
|
+
* for the happy path. Called from the `failedChunks === 0` branch of
|
|
4338
|
+
* `ingestDocument`. Runs INSIDE the caller's tx.
|
|
4339
|
+
*
|
|
4340
|
+
* Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
|
|
4341
|
+
* post-commit hooks (embedding lifecycle) AFTER the transaction commits.
|
|
4342
|
+
* On the partial path the caller never invokes this method; the empty
|
|
4343
|
+
* arrays stay empty.
|
|
4344
|
+
*/
|
|
4345
|
+
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
|
|
4346
|
+
const deletedSourceFactIds = [];
|
|
4347
|
+
const insertedFacts = [];
|
|
4348
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
4349
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
4350
|
+
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
4351
|
+
for (const existing of existingFacts) {
|
|
4352
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
4353
|
+
id: existing.id,
|
|
4354
|
+
okf_type: existing.okf_type ?? null
|
|
4355
|
+
});
|
|
4356
|
+
}
|
|
4357
|
+
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
4358
|
+
let { mode, manifest } = ontologyState;
|
|
4359
|
+
const hostNodes = [];
|
|
4360
|
+
const rawEdgeRequests = [];
|
|
4361
|
+
const now = Date.now();
|
|
4362
|
+
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
4363
|
+
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
4364
|
+
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
4365
|
+
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
4366
|
+
mode = ontologyState.mode;
|
|
4367
|
+
}
|
|
4368
|
+
for (const fact of facts) {
|
|
4369
|
+
const ontologyFact = fact;
|
|
4370
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
4371
|
+
const id = generateId("fact_");
|
|
4372
|
+
hostNodes.push({
|
|
4373
|
+
id,
|
|
4374
|
+
type: ontologyFact.okf_type ?? "",
|
|
4375
|
+
title: fact.title,
|
|
4376
|
+
body: fact.body,
|
|
4377
|
+
// Forward the LLM-extracted tags/confidence through the
|
|
4378
|
+
// extracted-shape upsertGraphCore path so the entries row
|
|
4379
|
+
// stores them (search filterability, heal-candidate
|
|
4380
|
+
// selection, runReembed embedding-text signal). The public
|
|
4381
|
+
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
4382
|
+
// — host-supplied deterministic nodes default to [] and
|
|
4383
|
+
// 'certain' as documented.
|
|
4384
|
+
tags: fact.tags,
|
|
4385
|
+
confidence: fact.confidence
|
|
4386
|
+
});
|
|
4387
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
4388
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
4389
|
+
if (normalized.edges.length > 0) {
|
|
4390
|
+
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
4391
|
+
}
|
|
4392
|
+
}
|
|
4393
|
+
}
|
|
4394
|
+
const hostEdges = [];
|
|
4395
|
+
for (const req of rawEdgeRequests) {
|
|
4396
|
+
const resolved = this.ontologyService?.resolveEdges(
|
|
4397
|
+
entityId,
|
|
4398
|
+
req.sourceId,
|
|
4399
|
+
req.sourceType,
|
|
4400
|
+
req.edges,
|
|
4401
|
+
manifest,
|
|
4402
|
+
titleIndex,
|
|
4403
|
+
now
|
|
4404
|
+
) ?? [];
|
|
4405
|
+
for (const e of resolved) {
|
|
4406
|
+
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
4407
|
+
}
|
|
4408
|
+
}
|
|
4409
|
+
await this.upsertGraphCore(
|
|
4410
|
+
entityId,
|
|
4411
|
+
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
4412
|
+
tx,
|
|
4413
|
+
{ strict: false }
|
|
4414
|
+
);
|
|
4415
|
+
return { deletedSourceFactIds, insertedFacts };
|
|
4416
|
+
}
|
|
4417
|
+
/**
|
|
4418
|
+
* Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
|
|
4419
|
+
* normalized title is not already in the live set for this (entityId,
|
|
4420
|
+
* sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
|
|
4421
|
+
* supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
|
|
4422
|
+
* ownership). Edges are NOT resolved on this path (no ontology-context
|
|
4423
|
+
* build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
|
|
4424
|
+
* prior attempts are recovered on the next full run's supersession.
|
|
4425
|
+
*
|
|
4426
|
+
* Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
|
|
4427
|
+
* reads from the most recently updated live row for the sourceRef, so
|
|
4428
|
+
* storing the incoming hash here would cause `hasChanged` to return
|
|
4429
|
+
* `false` on a same-hash retry — the failed chunks would never get a
|
|
4430
|
+
* second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
|
|
4431
|
+
* returns `true`, and the partial commit's sibling rows remain live
|
|
4432
|
+
* (deduped, not superseded) for the next attempt to extend.
|
|
4433
|
+
*
|
|
4434
|
+
* Runs INSIDE the caller's `tx`. Does not open a nested transaction.
|
|
4435
|
+
* Returns `{ inserted, skippedDuplicate }` for observability.
|
|
4436
|
+
*/
|
|
4437
|
+
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
|
|
4438
|
+
const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
|
|
4439
|
+
const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
|
|
4440
|
+
const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
|
|
4441
|
+
let inserted = 0;
|
|
4442
|
+
let skippedDuplicate = 0;
|
|
4443
|
+
const now = Date.now();
|
|
4444
|
+
const insertedDescriptors = [];
|
|
4445
|
+
for (const fact of dedupedFacts) {
|
|
4446
|
+
const normalizedTitle = normalizeTitleKey(fact.title);
|
|
4447
|
+
if (liveTitles.has(normalizedTitle)) {
|
|
4448
|
+
skippedDuplicate++;
|
|
4449
|
+
continue;
|
|
4450
|
+
}
|
|
4451
|
+
liveTitles.add(normalizedTitle);
|
|
4452
|
+
const id = generateId("fact_");
|
|
4453
|
+
const wikiFact = {
|
|
4454
|
+
id,
|
|
4455
|
+
entity_id: entityId,
|
|
4456
|
+
title: fact.title,
|
|
4457
|
+
body: fact.body,
|
|
4458
|
+
tags: fact.tags,
|
|
4459
|
+
confidence: fact.confidence,
|
|
4460
|
+
source_type: "immutable_document",
|
|
4461
|
+
// source_hash: null on partial rows — see method docstring. The
|
|
4462
|
+
// full path stamps the actual hash; partial rows stay hash-less so
|
|
4463
|
+
// `hasChanged` keeps returning true for retries.
|
|
4464
|
+
source_hash: null,
|
|
4465
|
+
source_ref: sourceRef,
|
|
4466
|
+
created_at: now,
|
|
4467
|
+
updated_at: now,
|
|
4468
|
+
last_accessed_at: null,
|
|
4469
|
+
access_count: 0,
|
|
4470
|
+
deleted_at: null,
|
|
4471
|
+
okf_type: null
|
|
4472
|
+
};
|
|
4473
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
4474
|
+
insertedDescriptors.push({
|
|
4475
|
+
id,
|
|
4476
|
+
entity_id: entityId,
|
|
4477
|
+
title: fact.title,
|
|
4478
|
+
body: fact.body,
|
|
4479
|
+
tags: JSON.stringify(fact.tags)
|
|
4480
|
+
});
|
|
4481
|
+
inserted++;
|
|
4482
|
+
}
|
|
4483
|
+
return { inserted, skippedDuplicate, insertedDescriptors };
|
|
4484
|
+
}
|
|
3845
4485
|
};
|
|
3846
4486
|
|
|
3847
4487
|
// src/services/BoundedLlmCall.ts
|
|
@@ -3962,6 +4602,25 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
|
3962
4602
|
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
3963
4603
|
var HEAL_BATCH_SIZE = 25;
|
|
3964
4604
|
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
4605
|
+
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
4606
|
+
var formatSkipError = (err) => {
|
|
4607
|
+
let base;
|
|
4608
|
+
if (err instanceof Error || typeof err !== "object" && typeof err !== "function") {
|
|
4609
|
+
base = safeErrorToString(err);
|
|
4610
|
+
} else {
|
|
4611
|
+
try {
|
|
4612
|
+
const json = JSON.stringify(err);
|
|
4613
|
+
base = typeof json === "string" ? json : safeErrorToString(err);
|
|
4614
|
+
} catch {
|
|
4615
|
+
base = safeErrorToString(err);
|
|
4616
|
+
}
|
|
4617
|
+
}
|
|
4618
|
+
if (base.length > SKIP_ERROR_LOG_CHARS) {
|
|
4619
|
+
const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
|
|
4620
|
+
return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
|
|
4621
|
+
}
|
|
4622
|
+
return base;
|
|
4623
|
+
};
|
|
3965
4624
|
var MaintenanceService = class {
|
|
3966
4625
|
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3967
4626
|
this.db = db;
|
|
@@ -4442,8 +5101,7 @@ var MaintenanceService = class {
|
|
|
4442
5101
|
maxPromptChars: HEAL_MAX_PROMPT_CHARS,
|
|
4443
5102
|
onSkip: (fact, err) => {
|
|
4444
5103
|
console.warn(
|
|
4445
|
-
`[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded
|
|
4446
|
-
err
|
|
5104
|
+
`[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
|
|
4447
5105
|
);
|
|
4448
5106
|
}
|
|
4449
5107
|
});
|
|
@@ -4581,8 +5239,7 @@ var MaintenanceService = class {
|
|
|
4581
5239
|
maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
|
|
4582
5240
|
onSkip: (fact, err) => {
|
|
4583
5241
|
console.warn(
|
|
4584
|
-
`[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded
|
|
4585
|
-
err
|
|
5242
|
+
`[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
|
|
4586
5243
|
);
|
|
4587
5244
|
}
|
|
4588
5245
|
});
|
|
@@ -4760,8 +5417,6 @@ var MaintenanceService = class {
|
|
|
4760
5417
|
return sanitizeRankerError(err, this.options.sanitizeRankerErrors);
|
|
4761
5418
|
}
|
|
4762
5419
|
};
|
|
4763
|
-
|
|
4764
|
-
// src/services/ImportExportService.ts
|
|
4765
5420
|
var MAX_EMBEDDING_BLOB_BYTES = 32 * 1024;
|
|
4766
5421
|
var IMPORT_TITLE_MAX = 500;
|
|
4767
5422
|
var IMPORT_BODY_MAX = 8e3;
|
|
@@ -4981,7 +5636,23 @@ var ImportExportService = class {
|
|
|
4981
5636
|
access_count: fact.access_count,
|
|
4982
5637
|
deleted_at: fact.deleted_at,
|
|
4983
5638
|
embedding_blob: blobData ?? void 0,
|
|
4984
|
-
okf_type: fact.okf_type ?? null
|
|
5639
|
+
okf_type: fact.okf_type ?? null,
|
|
5640
|
+
// OKF v0.2 — forwarded as-is; parseOkfBundle / the caller is responsible for
|
|
5641
|
+
// populating these per spec, importDump does not derive or validate them.
|
|
5642
|
+
lifecycle_status: fact.lifecycle_status ?? "stable",
|
|
5643
|
+
stale_after: fact.stale_after ?? null,
|
|
5644
|
+
generated_by: fact.generated_by ?? null,
|
|
5645
|
+
okf_sources: fact.okf_sources ?? [],
|
|
5646
|
+
okf_verified: fact.okf_verified ?? [],
|
|
5647
|
+
okf_usage_window: fact.okf_usage_window ?? null,
|
|
5648
|
+
// last_verified_at/_by are DERIVED mirrors of `okf_verified` per spec
|
|
5649
|
+
// §5.3. Recomputing here (instead of trusting the payload field) closes
|
|
5650
|
+
// the invariant that a raw importDump cannot carry a
|
|
5651
|
+
// last_verified_by that contradicts its own okf_verified list.
|
|
5652
|
+
...(() => {
|
|
5653
|
+
const latest = coreOkf.latestVerified(fact.okf_verified ?? [], Date.now());
|
|
5654
|
+
return { last_verified_at: latest?.at ?? null, last_verified_by: latest?.by ?? null };
|
|
5655
|
+
})()
|
|
4985
5656
|
};
|
|
4986
5657
|
await this.entryRepo.upsertForImport(factObj, tx);
|
|
4987
5658
|
if (blobData != null) {
|
|
@@ -5031,7 +5702,19 @@ var ImportExportService = class {
|
|
|
5031
5702
|
updated_at: safeUpdatedAt,
|
|
5032
5703
|
resolved_at: task.resolved_at,
|
|
5033
5704
|
deleted_at: task.deleted_at,
|
|
5034
|
-
okf_type: task.okf_type ?? null
|
|
5705
|
+
okf_type: task.okf_type ?? null,
|
|
5706
|
+
// OKF v0.2
|
|
5707
|
+
lifecycle_status: task.lifecycle_status ?? "stable",
|
|
5708
|
+
stale_after: task.stale_after ?? null,
|
|
5709
|
+
generated_by: task.generated_by ?? null,
|
|
5710
|
+
okf_sources: task.okf_sources ?? [],
|
|
5711
|
+
okf_verified: task.okf_verified ?? [],
|
|
5712
|
+
okf_usage_window: task.okf_usage_window ?? null,
|
|
5713
|
+
// Derived mirrors — see fact branch above; same invariant.
|
|
5714
|
+
...(() => {
|
|
5715
|
+
const latest = coreOkf.latestVerified(task.okf_verified ?? [], Date.now());
|
|
5716
|
+
return { last_verified_at: latest?.at ?? null, last_verified_by: latest?.by ?? null };
|
|
5717
|
+
})()
|
|
5035
5718
|
},
|
|
5036
5719
|
tx,
|
|
5037
5720
|
safeUpdatedAt
|
|
@@ -6157,6 +6840,164 @@ var GraphTraversalService = class {
|
|
|
6157
6840
|
return { nodes, edges: filteredEdges };
|
|
6158
6841
|
}
|
|
6159
6842
|
};
|
|
6843
|
+
var OkfTrustWritesRepository = class {
|
|
6844
|
+
constructor(db, prefix) {
|
|
6845
|
+
this.db = db;
|
|
6846
|
+
this.prefix = prefix;
|
|
6847
|
+
}
|
|
6848
|
+
getExecutor(tx) {
|
|
6849
|
+
return tx ?? this.db;
|
|
6850
|
+
}
|
|
6851
|
+
/**
|
|
6852
|
+
* Append one or more verification events to a fact's `okf_verified` JSON
|
|
6853
|
+
* array, atomically, in a single SQL statement.
|
|
6854
|
+
*
|
|
6855
|
+
* The implementation uses json_each over the existing array UNION ALL'd
|
|
6856
|
+
* with json_each over the new events wrapped in json_group_array — a single
|
|
6857
|
+
* UPDATE expression. SQLite serializes this against concurrent writers on
|
|
6858
|
+
* the same row, so two callers appending simultaneously cannot lose each
|
|
6859
|
+
* other's events (the previous implementation serialized only the supplied
|
|
6860
|
+
* array and overwrote, dropping any pre-existing history).
|
|
6861
|
+
*
|
|
6862
|
+
* Sets the `last_verified_*` convenience columns from the latest event in
|
|
6863
|
+
* the union (computed in code; see {@link latestVerified}).
|
|
6864
|
+
*/
|
|
6865
|
+
async writeOkfTrust(entryId, entityId, verified, tx) {
|
|
6866
|
+
const executor = this.getExecutor(tx);
|
|
6867
|
+
if (verified.length === 0) return;
|
|
6868
|
+
const latest = coreOkf.latestVerified(verified, Date.now());
|
|
6869
|
+
const newEventsJson = JSON.stringify(verified);
|
|
6870
|
+
await executor.runAsync(
|
|
6871
|
+
`UPDATE ${this.prefix}entries
|
|
6872
|
+
SET okf_verified = COALESCE(
|
|
6873
|
+
(SELECT json_group_array(json_object('by', s.by_val, 'at', s.at_val))
|
|
6874
|
+
FROM (
|
|
6875
|
+
SELECT json_extract(value, '$.by') AS by_val,
|
|
6876
|
+
json_extract(value, '$.at') AS at_val
|
|
6877
|
+
FROM json_each(okf_verified)
|
|
6878
|
+
UNION ALL
|
|
6879
|
+
SELECT json_extract(value, '$.by'),
|
|
6880
|
+
json_extract(value, '$.at')
|
|
6881
|
+
FROM json_each(?)
|
|
6882
|
+
) AS s
|
|
6883
|
+
ORDER BY s.at_val),
|
|
6884
|
+
'[]'
|
|
6885
|
+
),
|
|
6886
|
+
last_verified_by = ?,
|
|
6887
|
+
last_verified_at = ?
|
|
6888
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6889
|
+
[newEventsJson, latest?.by ?? null, latest?.at ?? null, entryId, entityId]
|
|
6890
|
+
);
|
|
6891
|
+
}
|
|
6892
|
+
/** Replace the `sources` list (provenance). */
|
|
6893
|
+
async writeOkfSources(entryId, entityId, sources, tx) {
|
|
6894
|
+
const executor = this.getExecutor(tx);
|
|
6895
|
+
const json = coreOkf.formatSourcesJson(sources);
|
|
6896
|
+
await executor.runAsync(
|
|
6897
|
+
`UPDATE ${this.prefix}entries
|
|
6898
|
+
SET okf_sources = ?
|
|
6899
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6900
|
+
[json, entryId, entityId]
|
|
6901
|
+
);
|
|
6902
|
+
}
|
|
6903
|
+
/** Set the OKF v0.2 lifecycle status. */
|
|
6904
|
+
async setLifecycleStatus(entryId, entityId, status, tx) {
|
|
6905
|
+
const executor = this.getExecutor(tx);
|
|
6906
|
+
await executor.runAsync(
|
|
6907
|
+
`UPDATE ${this.prefix}entries
|
|
6908
|
+
SET lifecycle_status = ?
|
|
6909
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6910
|
+
[status, entryId, entityId]
|
|
6911
|
+
);
|
|
6912
|
+
}
|
|
6913
|
+
/** Set the absolute cutoff (epoch ms) for staleness, or NULL to clear. */
|
|
6914
|
+
async setStaleAfter(entryId, entityId, date, tx) {
|
|
6915
|
+
const executor = this.getExecutor(tx);
|
|
6916
|
+
await executor.runAsync(
|
|
6917
|
+
`UPDATE ${this.prefix}entries
|
|
6918
|
+
SET stale_after = ?
|
|
6919
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6920
|
+
[date, entryId, entityId]
|
|
6921
|
+
);
|
|
6922
|
+
}
|
|
6923
|
+
/** Set the producer / verifier actor string. Rare. */
|
|
6924
|
+
async setGeneratedBy(entryId, entityId, actor, tx) {
|
|
6925
|
+
const executor = this.getExecutor(tx);
|
|
6926
|
+
await executor.runAsync(
|
|
6927
|
+
`UPDATE ${this.prefix}entries
|
|
6928
|
+
SET generated_by = ?
|
|
6929
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6930
|
+
[actor, entryId, entityId]
|
|
6931
|
+
);
|
|
6932
|
+
}
|
|
6933
|
+
// The same five methods exist for tasks, with `tasks` instead of `entries`.
|
|
6934
|
+
// Tasks are symmetric per spec §2.5.
|
|
6935
|
+
/** Atomic append, task variant — mirrors {@link writeOkfTrust}. */
|
|
6936
|
+
async writeOkfTrustTask(taskId, entityId, verified, tx) {
|
|
6937
|
+
const executor = this.getExecutor(tx);
|
|
6938
|
+
if (verified.length === 0) return;
|
|
6939
|
+
const latest = coreOkf.latestVerified(verified, Date.now());
|
|
6940
|
+
const newEventsJson = JSON.stringify(verified);
|
|
6941
|
+
await executor.runAsync(
|
|
6942
|
+
`UPDATE ${this.prefix}tasks
|
|
6943
|
+
SET okf_verified = COALESCE(
|
|
6944
|
+
(SELECT json_group_array(json_object('by', s.by_val, 'at', s.at_val))
|
|
6945
|
+
FROM (
|
|
6946
|
+
SELECT json_extract(value, '$.by') AS by_val,
|
|
6947
|
+
json_extract(value, '$.at') AS at_val
|
|
6948
|
+
FROM json_each(okf_verified)
|
|
6949
|
+
UNION ALL
|
|
6950
|
+
SELECT json_extract(value, '$.by'),
|
|
6951
|
+
json_extract(value, '$.at')
|
|
6952
|
+
FROM json_each(?)
|
|
6953
|
+
) AS s
|
|
6954
|
+
ORDER BY s.at_val),
|
|
6955
|
+
'[]'
|
|
6956
|
+
),
|
|
6957
|
+
last_verified_by = ?,
|
|
6958
|
+
last_verified_at = ?
|
|
6959
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6960
|
+
[newEventsJson, latest?.by ?? null, latest?.at ?? null, taskId, entityId]
|
|
6961
|
+
);
|
|
6962
|
+
}
|
|
6963
|
+
async writeOkfSourcesTask(taskId, entityId, sources, tx) {
|
|
6964
|
+
const executor = this.getExecutor(tx);
|
|
6965
|
+
const json = coreOkf.formatSourcesJson(sources);
|
|
6966
|
+
await executor.runAsync(
|
|
6967
|
+
`UPDATE ${this.prefix}tasks
|
|
6968
|
+
SET okf_sources = ?
|
|
6969
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6970
|
+
[json, taskId, entityId]
|
|
6971
|
+
);
|
|
6972
|
+
}
|
|
6973
|
+
async setLifecycleStatusTask(taskId, entityId, status, tx) {
|
|
6974
|
+
const executor = this.getExecutor(tx);
|
|
6975
|
+
await executor.runAsync(
|
|
6976
|
+
`UPDATE ${this.prefix}tasks
|
|
6977
|
+
SET lifecycle_status = ?
|
|
6978
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6979
|
+
[status, taskId, entityId]
|
|
6980
|
+
);
|
|
6981
|
+
}
|
|
6982
|
+
async setStaleAfterTask(taskId, entityId, date, tx) {
|
|
6983
|
+
const executor = this.getExecutor(tx);
|
|
6984
|
+
await executor.runAsync(
|
|
6985
|
+
`UPDATE ${this.prefix}tasks
|
|
6986
|
+
SET stale_after = ?
|
|
6987
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6988
|
+
[date, taskId, entityId]
|
|
6989
|
+
);
|
|
6990
|
+
}
|
|
6991
|
+
async setGeneratedByTask(taskId, entityId, actor, tx) {
|
|
6992
|
+
const executor = this.getExecutor(tx);
|
|
6993
|
+
await executor.runAsync(
|
|
6994
|
+
`UPDATE ${this.prefix}tasks
|
|
6995
|
+
SET generated_by = ?
|
|
6996
|
+
WHERE id = ? AND entity_id = ?`,
|
|
6997
|
+
[actor, taskId, entityId]
|
|
6998
|
+
);
|
|
6999
|
+
}
|
|
7000
|
+
};
|
|
6160
7001
|
|
|
6161
7002
|
// src/WikiMemory.ts
|
|
6162
7003
|
var TABLE_PREFIX_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,30}_$/;
|
|
@@ -6251,6 +7092,7 @@ var WikiMemory = class {
|
|
|
6251
7092
|
this.entryRepo,
|
|
6252
7093
|
this.options.config ?? {}
|
|
6253
7094
|
);
|
|
7095
|
+
this.okfTrustWrites = new OkfTrustWritesRepository(this.db, this.prefix);
|
|
6254
7096
|
}
|
|
6255
7097
|
/**
|
|
6256
7098
|
* Explicit escape hatch for test suites: typed access to composed services for mocks/spies.
|
|
@@ -6618,6 +7460,42 @@ var WikiMemory = class {
|
|
|
6618
7460
|
);
|
|
6619
7461
|
this.ontologyService.invalidateCache(entityId);
|
|
6620
7462
|
}
|
|
7463
|
+
/** Append a verification event to a fact. Does NOT touch `updated_at`. */
|
|
7464
|
+
async writeOkfTrust(entryId, entityId, verified) {
|
|
7465
|
+
return this.okfTrustWrites.writeOkfTrust(entryId, entityId, verified);
|
|
7466
|
+
}
|
|
7467
|
+
/** Replace a fact's source list. Does NOT touch `updated_at`. */
|
|
7468
|
+
async writeOkfSources(entryId, entityId, sources) {
|
|
7469
|
+
return this.okfTrustWrites.writeOkfSources(entryId, entityId, sources);
|
|
7470
|
+
}
|
|
7471
|
+
/** Set a fact's OKF v0.2 lifecycle status. Does NOT touch `updated_at`. */
|
|
7472
|
+
async setLifecycleStatus(entryId, entityId, status) {
|
|
7473
|
+
return this.okfTrustWrites.setLifecycleStatus(entryId, entityId, status);
|
|
7474
|
+
}
|
|
7475
|
+
/** Set a fact's stale_after (epoch ms) or clear it. Does NOT touch `updated_at`. */
|
|
7476
|
+
async setStaleAfter(entryId, entityId, date) {
|
|
7477
|
+
return this.okfTrustWrites.setStaleAfter(entryId, entityId, date);
|
|
7478
|
+
}
|
|
7479
|
+
/** Set a fact's generated_by actor string. Does NOT touch `updated_at`. */
|
|
7480
|
+
async setGeneratedBy(entryId, entityId, actor) {
|
|
7481
|
+
return this.okfTrustWrites.setGeneratedBy(entryId, entityId, actor);
|
|
7482
|
+
}
|
|
7483
|
+
/** Task variants of the same five DAO methods. Symmetric per spec §2.5. */
|
|
7484
|
+
async writeOkfTrustTask(taskId, entityId, verified) {
|
|
7485
|
+
return this.okfTrustWrites.writeOkfTrustTask(taskId, entityId, verified);
|
|
7486
|
+
}
|
|
7487
|
+
async writeOkfSourcesTask(taskId, entityId, sources) {
|
|
7488
|
+
return this.okfTrustWrites.writeOkfSourcesTask(taskId, entityId, sources);
|
|
7489
|
+
}
|
|
7490
|
+
async setLifecycleStatusTask(taskId, entityId, status) {
|
|
7491
|
+
return this.okfTrustWrites.setLifecycleStatusTask(taskId, entityId, status);
|
|
7492
|
+
}
|
|
7493
|
+
async setStaleAfterTask(taskId, entityId, date) {
|
|
7494
|
+
return this.okfTrustWrites.setStaleAfterTask(taskId, entityId, date);
|
|
7495
|
+
}
|
|
7496
|
+
async setGeneratedByTask(taskId, entityId, actor) {
|
|
7497
|
+
return this.okfTrustWrites.setGeneratedByTask(taskId, entityId, actor);
|
|
7498
|
+
}
|
|
6621
7499
|
};
|
|
6622
7500
|
_testAccessNonTestEnvWarned = new WeakMap();
|
|
6623
7501
|
|
|
@@ -6921,13 +7799,87 @@ function formatMemoryDump(dump) {
|
|
|
6921
7799
|
files
|
|
6922
7800
|
};
|
|
6923
7801
|
}
|
|
6924
|
-
var
|
|
6925
|
-
function
|
|
7802
|
+
var DEFAULT_PROFILE = "llm-wiki/2";
|
|
7803
|
+
function isoOrFallback(updatedAt) {
|
|
7804
|
+
if (!Number.isFinite(updatedAt)) return (/* @__PURE__ */ new Date(0)).toISOString();
|
|
7805
|
+
const d = new Date(updatedAt);
|
|
7806
|
+
if (!Number.isFinite(d.getTime())) return (/* @__PURE__ */ new Date(0)).toISOString();
|
|
7807
|
+
return d.toISOString();
|
|
7808
|
+
}
|
|
7809
|
+
function isoDateOrOmit(ms) {
|
|
7810
|
+
if (ms == null || !Number.isFinite(ms)) return void 0;
|
|
7811
|
+
const d = new Date(ms);
|
|
7812
|
+
if (!Number.isFinite(d.getTime())) return void 0;
|
|
7813
|
+
return d.toISOString().slice(0, 10);
|
|
7814
|
+
}
|
|
7815
|
+
function factFrontmatterV2(f) {
|
|
6926
7816
|
return {
|
|
6927
7817
|
type: f.okf_type ?? "fact",
|
|
6928
7818
|
title: f.title,
|
|
6929
7819
|
tags: f.tags,
|
|
6930
|
-
timestamp
|
|
7820
|
+
// `timestamp` rides along deliberately even on the v0.2 path — a naive
|
|
7821
|
+
// consumer that only understands the v0.1 key still gets a readable date
|
|
7822
|
+
// without needing flow-mapping support. `generated.at` is the canonical
|
|
7823
|
+
// v0.2 field and always wins on import when both are present (spec §2.4).
|
|
7824
|
+
timestamp: isoOrFallback(f.updated_at),
|
|
7825
|
+
resource: f.source_ref ?? void 0,
|
|
7826
|
+
id: f.id,
|
|
7827
|
+
entity_id: f.entity_id,
|
|
7828
|
+
confidence: f.confidence,
|
|
7829
|
+
source_type: f.source_type,
|
|
7830
|
+
source_hash: f.source_hash,
|
|
7831
|
+
created_at: f.created_at,
|
|
7832
|
+
access_count: f.access_count,
|
|
7833
|
+
last_accessed_at: f.last_accessed_at,
|
|
7834
|
+
deleted_at: f.deleted_at,
|
|
7835
|
+
// OKF v0.2
|
|
7836
|
+
status: f.lifecycle_status ?? "stable",
|
|
7837
|
+
...isoDateOrOmit(f.stale_after) ? { stale_after: isoDateOrOmit(f.stale_after) } : {},
|
|
7838
|
+
// Omit `generated` entirely when there's no actor on record — never
|
|
7839
|
+
// fabricate a `generated_by` (mirrors the import side's rule in §4.8:
|
|
7840
|
+
// it never invents one either, so export must not manufacture one that
|
|
7841
|
+
// would then round-trip back in as if it had always been asserted).
|
|
7842
|
+
...f.generated_by ? { generated: { by: f.generated_by, at: isoOrFallback(f.updated_at) } } : {},
|
|
7843
|
+
// Omit `verified`/`sources` when empty — an emitted `[]` is not the same
|
|
7844
|
+
// shape as an absent key (spec §5.3 derives `unverified` from "no
|
|
7845
|
+
// `verified` key", not from "empty `verified` list").
|
|
7846
|
+
...f.okf_verified && f.okf_verified.length > 0 ? { verified: f.okf_verified } : {},
|
|
7847
|
+
...f.okf_sources && f.okf_sources.length > 0 ? { sources: f.okf_sources } : {},
|
|
7848
|
+
...f.okf_usage_window ? { usage_window: f.okf_usage_window } : {}
|
|
7849
|
+
};
|
|
7850
|
+
}
|
|
7851
|
+
function taskFrontmatterV2(t) {
|
|
7852
|
+
return {
|
|
7853
|
+
type: t.okf_type ?? "task",
|
|
7854
|
+
title: t.description,
|
|
7855
|
+
timestamp: isoOrFallback(t.updated_at),
|
|
7856
|
+
// see factFrontmatterV2 comment — deliberate back-compat duplication
|
|
7857
|
+
id: t.id,
|
|
7858
|
+
entity_id: t.entity_id,
|
|
7859
|
+
status: t.lifecycle_status ?? "stable",
|
|
7860
|
+
// OKF v0.2 lifecycle
|
|
7861
|
+
execution_status: t.status,
|
|
7862
|
+
// execution state under renamed key
|
|
7863
|
+
priority: t.priority,
|
|
7864
|
+
created_at: t.created_at,
|
|
7865
|
+
resolved_at: t.resolved_at,
|
|
7866
|
+
deleted_at: t.deleted_at,
|
|
7867
|
+
// OKF v0.2 — symmetric with facts (spec §2.5): stale_after applies to
|
|
7868
|
+
// tasks too (a hallucinated task needs a freshness signal as much as a
|
|
7869
|
+
// hallucinated fact does).
|
|
7870
|
+
...isoDateOrOmit(t.stale_after) ? { stale_after: isoDateOrOmit(t.stale_after) } : {},
|
|
7871
|
+
...t.generated_by ? { generated: { by: t.generated_by, at: isoOrFallback(t.updated_at) } } : {},
|
|
7872
|
+
...t.okf_verified && t.okf_verified.length > 0 ? { verified: t.okf_verified } : {},
|
|
7873
|
+
...t.okf_sources && t.okf_sources.length > 0 ? { sources: t.okf_sources } : {},
|
|
7874
|
+
...t.okf_usage_window ? { usage_window: t.okf_usage_window } : {}
|
|
7875
|
+
};
|
|
7876
|
+
}
|
|
7877
|
+
function factFrontmatterV1(f) {
|
|
7878
|
+
return {
|
|
7879
|
+
type: f.okf_type ?? "fact",
|
|
7880
|
+
title: f.title,
|
|
7881
|
+
tags: f.tags,
|
|
7882
|
+
timestamp: isoOrFallback(f.updated_at),
|
|
6931
7883
|
resource: f.source_ref ?? void 0,
|
|
6932
7884
|
id: f.id,
|
|
6933
7885
|
entity_id: f.entity_id,
|
|
@@ -6940,11 +7892,11 @@ function factFrontmatter(f) {
|
|
|
6940
7892
|
deleted_at: f.deleted_at
|
|
6941
7893
|
};
|
|
6942
7894
|
}
|
|
6943
|
-
function
|
|
7895
|
+
function taskFrontmatterV1(t) {
|
|
6944
7896
|
return {
|
|
6945
7897
|
type: t.okf_type ?? "task",
|
|
6946
7898
|
title: t.description,
|
|
6947
|
-
timestamp:
|
|
7899
|
+
timestamp: isoOrFallback(t.updated_at),
|
|
6948
7900
|
id: t.id,
|
|
6949
7901
|
entity_id: t.entity_id,
|
|
6950
7902
|
status: t.status,
|
|
@@ -6974,7 +7926,11 @@ function buildEventLogEntries(events, factIdToFilename) {
|
|
|
6974
7926
|
return { date: formatLogDate(e.created_at), text: coreOkf.appendEventIdComment(text, e.id) };
|
|
6975
7927
|
});
|
|
6976
7928
|
}
|
|
6977
|
-
function formatOkfBundle(dump) {
|
|
7929
|
+
function formatOkfBundle(dump, options) {
|
|
7930
|
+
const profile = options?.profile ?? DEFAULT_PROFILE;
|
|
7931
|
+
const okfVersion = profile === "llm-wiki/2" ? "0.2" : "0.1";
|
|
7932
|
+
const factFm = profile === "llm-wiki/2" ? factFrontmatterV2 : factFrontmatterV1;
|
|
7933
|
+
const taskFm = profile === "llm-wiki/2" ? taskFrontmatterV2 : taskFrontmatterV1;
|
|
6978
7934
|
const files = [];
|
|
6979
7935
|
const rootEntries = [];
|
|
6980
7936
|
for (const [entityId, bundle] of Object.entries(dump.entities)) {
|
|
@@ -7011,7 +7967,7 @@ function formatOkfBundle(dump) {
|
|
|
7011
7967
|
const factBody = coreOkf.appendRelatedSection(f.body, relatedLinksFor2(factPath, f.id));
|
|
7012
7968
|
files.push({
|
|
7013
7969
|
path: factPath,
|
|
7014
|
-
content: coreOkf.buildConceptDocument(
|
|
7970
|
+
content: coreOkf.buildConceptDocument(factFm(f), factBody)
|
|
7015
7971
|
});
|
|
7016
7972
|
}
|
|
7017
7973
|
const taskEntries = bundle.tasks.map((t) => ({
|
|
@@ -7023,7 +7979,7 @@ function formatOkfBundle(dump) {
|
|
|
7023
7979
|
const taskBody = coreOkf.appendRelatedSection("", relatedLinksFor2(taskPath, t.id));
|
|
7024
7980
|
files.push({
|
|
7025
7981
|
path: taskPath,
|
|
7026
|
-
content: coreOkf.buildConceptDocument(
|
|
7982
|
+
content: coreOkf.buildConceptDocument(taskFm(t), taskBody)
|
|
7027
7983
|
});
|
|
7028
7984
|
}
|
|
7029
7985
|
files.push({
|
|
@@ -7046,9 +8002,9 @@ function formatOkfBundle(dump) {
|
|
|
7046
8002
|
files.push({
|
|
7047
8003
|
path: "index.md",
|
|
7048
8004
|
content: coreOkf.buildRootIndexMd(
|
|
7049
|
-
|
|
8005
|
+
okfVersion,
|
|
7050
8006
|
rootEntries.length > 0 ? [{ heading: "Entities", entries: rootEntries }] : [],
|
|
7051
|
-
{ profile
|
|
8007
|
+
{ profile }
|
|
7052
8008
|
)
|
|
7053
8009
|
});
|
|
7054
8010
|
return { files };
|
|
@@ -7138,6 +8094,21 @@ function parseFrontmatterTimestamp(value, fallback) {
|
|
|
7138
8094
|
}
|
|
7139
8095
|
return fallback;
|
|
7140
8096
|
}
|
|
8097
|
+
function parseStaleAfterRaw(raw) {
|
|
8098
|
+
if (typeof raw !== "string") return null;
|
|
8099
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(raw);
|
|
8100
|
+
if (!m) return null;
|
|
8101
|
+
const year = Number(m[1]);
|
|
8102
|
+
const month = Number(m[2]);
|
|
8103
|
+
const day = Number(m[3]);
|
|
8104
|
+
const ms = Date.UTC(year, month - 1, day);
|
|
8105
|
+
if (!Number.isFinite(ms)) return null;
|
|
8106
|
+
const d = new Date(ms);
|
|
8107
|
+
if (d.getUTCFullYear() !== year || d.getUTCMonth() + 1 !== month || d.getUTCDate() !== day) {
|
|
8108
|
+
return null;
|
|
8109
|
+
}
|
|
8110
|
+
return ms;
|
|
8111
|
+
}
|
|
7141
8112
|
function unescapeLogSummary(summary) {
|
|
7142
8113
|
return summary.replace(/\\\]/g, "]").replace(/\\\[/g, "[").replace(/\\\\/g, "\\");
|
|
7143
8114
|
}
|
|
@@ -7155,9 +8126,29 @@ function parseLogEntryText(text) {
|
|
|
7155
8126
|
function frontmatterToFact(entityId, id, frontmatter, body, now) {
|
|
7156
8127
|
const created_at = parseFrontmatterTimestamp(frontmatter.created_at, now);
|
|
7157
8128
|
const updated_at = parseFrontmatterTimestamp(
|
|
7158
|
-
frontmatter.
|
|
7159
|
-
parseFrontmatterTimestamp(
|
|
8129
|
+
frontmatter.generated?.at,
|
|
8130
|
+
parseFrontmatterTimestamp(
|
|
8131
|
+
frontmatter.timestamp,
|
|
8132
|
+
parseFrontmatterTimestamp(frontmatter.updated_at, now)
|
|
8133
|
+
)
|
|
7160
8134
|
);
|
|
8135
|
+
const verified = coreOkf.parseVerifiedFlexible(frontmatter.verified);
|
|
8136
|
+
const rawSources = frontmatter.sources;
|
|
8137
|
+
const hasSourcesKey = rawSources !== void 0;
|
|
8138
|
+
const sources = Array.isArray(rawSources) ? rawSources : [];
|
|
8139
|
+
const usageWindow = frontmatter.usage_window && typeof frontmatter.usage_window === "object" ? frontmatter.usage_window : null;
|
|
8140
|
+
const lastV = coreOkf.latestVerified(verified, now);
|
|
8141
|
+
const lifecycleRaw = frontmatter.status;
|
|
8142
|
+
const lifecycle_status = lifecycleRaw === "draft" || lifecycleRaw === "stable" || lifecycleRaw === "deprecated" ? lifecycleRaw : "stable";
|
|
8143
|
+
const staleAfterRaw = frontmatter.stale_after;
|
|
8144
|
+
const stale_after = parseStaleAfterRaw(staleAfterRaw);
|
|
8145
|
+
const generated_by = frontmatter.generated?.by ?? null;
|
|
8146
|
+
if (!hasSourcesKey) {
|
|
8147
|
+
const urls = coreOkf.parseCitationsList(body);
|
|
8148
|
+
for (const url of urls) {
|
|
8149
|
+
sources.push({ resource: url });
|
|
8150
|
+
}
|
|
8151
|
+
}
|
|
7161
8152
|
return {
|
|
7162
8153
|
id,
|
|
7163
8154
|
entity_id: entityId,
|
|
@@ -7173,26 +8164,62 @@ function frontmatterToFact(entityId, id, frontmatter, body, now) {
|
|
|
7173
8164
|
last_accessed_at: frontmatter.last_accessed_at != null ? parseFrontmatterTimestamp(frontmatter.last_accessed_at, now) : null,
|
|
7174
8165
|
access_count: typeof frontmatter.access_count === "number" ? frontmatter.access_count : 0,
|
|
7175
8166
|
deleted_at: frontmatter.deleted_at != null ? parseFrontmatterTimestamp(frontmatter.deleted_at, 0) : null,
|
|
7176
|
-
okf_type: frontmatter.type
|
|
8167
|
+
okf_type: frontmatter.type,
|
|
8168
|
+
// OKF v0.2
|
|
8169
|
+
lifecycle_status,
|
|
8170
|
+
stale_after,
|
|
8171
|
+
generated_by,
|
|
8172
|
+
okf_sources: sources,
|
|
8173
|
+
okf_verified: verified,
|
|
8174
|
+
okf_usage_window: usageWindow,
|
|
8175
|
+
last_verified_at: lastV?.at ?? null,
|
|
8176
|
+
last_verified_by: lastV?.by ?? null
|
|
7177
8177
|
};
|
|
7178
8178
|
}
|
|
7179
|
-
function frontmatterToTask(entityId, id, frontmatter, now) {
|
|
8179
|
+
function frontmatterToTask(entityId, id, frontmatter, now, usesV1StatusRule) {
|
|
7180
8180
|
const created_at = parseFrontmatterTimestamp(frontmatter.created_at, now);
|
|
7181
8181
|
const updated_at = parseFrontmatterTimestamp(
|
|
7182
|
-
frontmatter.
|
|
7183
|
-
parseFrontmatterTimestamp(
|
|
8182
|
+
frontmatter.generated?.at,
|
|
8183
|
+
parseFrontmatterTimestamp(
|
|
8184
|
+
frontmatter.timestamp,
|
|
8185
|
+
parseFrontmatterTimestamp(frontmatter.updated_at, now)
|
|
8186
|
+
)
|
|
7184
8187
|
);
|
|
8188
|
+
const verified = coreOkf.parseVerifiedFlexible(frontmatter.verified);
|
|
8189
|
+
const sources = Array.isArray(frontmatter.sources) ? frontmatter.sources : [];
|
|
8190
|
+
const usageWindow = frontmatter.usage_window && typeof frontmatter.usage_window === "object" ? frontmatter.usage_window : null;
|
|
8191
|
+
const lastV = coreOkf.latestVerified(verified, now);
|
|
8192
|
+
const lifecycleRaw = frontmatter.status;
|
|
8193
|
+
const lifecycle_status = lifecycleRaw === "draft" || lifecycleRaw === "stable" || lifecycleRaw === "deprecated" ? lifecycleRaw : "stable";
|
|
8194
|
+
const staleAfterRaw = frontmatter.stale_after;
|
|
8195
|
+
const stale_after = parseStaleAfterRaw(staleAfterRaw);
|
|
8196
|
+
const executionRaw = frontmatter.execution_status;
|
|
8197
|
+
let executionState;
|
|
8198
|
+
if (usesV1StatusRule) {
|
|
8199
|
+
executionState = TASK_STATUSES.has(String(frontmatter.status)) ? frontmatter.status : "pending";
|
|
8200
|
+
} else {
|
|
8201
|
+
executionState = TASK_STATUSES.has(String(executionRaw)) ? executionRaw : TASK_STATUSES.has(String(frontmatter.status)) ? frontmatter.status : "pending";
|
|
8202
|
+
}
|
|
7185
8203
|
return {
|
|
7186
8204
|
id,
|
|
7187
8205
|
entity_id: entityId,
|
|
7188
8206
|
description: typeof frontmatter.title === "string" ? frontmatter.title : "",
|
|
7189
|
-
status:
|
|
8207
|
+
status: executionState,
|
|
7190
8208
|
priority: typeof frontmatter.priority === "number" ? frontmatter.priority : 0,
|
|
7191
8209
|
created_at,
|
|
7192
8210
|
updated_at,
|
|
7193
8211
|
resolved_at: frontmatter.resolved_at != null ? parseFrontmatterTimestamp(frontmatter.resolved_at, now) : null,
|
|
7194
8212
|
deleted_at: frontmatter.deleted_at != null ? parseFrontmatterTimestamp(frontmatter.deleted_at, 0) : null,
|
|
7195
|
-
okf_type: frontmatter.type
|
|
8213
|
+
okf_type: frontmatter.type,
|
|
8214
|
+
// OKF v0.2
|
|
8215
|
+
lifecycle_status,
|
|
8216
|
+
stale_after,
|
|
8217
|
+
generated_by: frontmatter.generated?.by ?? null,
|
|
8218
|
+
okf_sources: sources,
|
|
8219
|
+
okf_verified: verified,
|
|
8220
|
+
okf_usage_window: usageWindow,
|
|
8221
|
+
last_verified_at: lastV?.at ?? null,
|
|
8222
|
+
last_verified_by: lastV?.by ?? null
|
|
7196
8223
|
};
|
|
7197
8224
|
}
|
|
7198
8225
|
function findLogMdPath(files) {
|
|
@@ -7239,10 +8266,17 @@ function parseOkfBundle(entityId, files, options) {
|
|
|
7239
8266
|
);
|
|
7240
8267
|
const rootIndex = allowedFiles.find((f) => f.path === "index.md");
|
|
7241
8268
|
const profileMeta = rootIndex ? coreOkf.parseRootIndexMd(rootIndex.content) : {};
|
|
7242
|
-
const
|
|
8269
|
+
const okfVersion = profileMeta.okf_version;
|
|
8270
|
+
const profile = profileMeta.profile;
|
|
8271
|
+
const isProfile1 = profile === "llm-wiki/1";
|
|
8272
|
+
const isProfile2 = profile === "llm-wiki/2";
|
|
8273
|
+
const isLegacyV1 = profile === void 0 && okfVersion === "0.1";
|
|
8274
|
+
const isVersionOnlyV2 = profile === void 0 && okfVersion === "0.2";
|
|
8275
|
+
const isUnknownProfileV2 = profile !== void 0 && profile !== "llm-wiki/1" && profile !== "llm-wiki/2" && okfVersion === "0.2";
|
|
8276
|
+
const summaryEligible = isProfile1 || isProfile2 || isLegacyV1 || isVersionOnlyV2 || isUnknownProfileV2;
|
|
7243
8277
|
let entitySummary;
|
|
7244
8278
|
const entityIndex = allowedFiles.find((f) => f.path === `${entityPrefix}index.md`);
|
|
7245
|
-
if (
|
|
8279
|
+
if (summaryEligible && entityIndex) {
|
|
7246
8280
|
const summary = coreOkf.parseEntityIndexMd(entityIndex.content).summary;
|
|
7247
8281
|
entitySummary = summary || void 0;
|
|
7248
8282
|
}
|
|
@@ -7271,11 +8305,12 @@ function parseOkfBundle(entityId, files, options) {
|
|
|
7271
8305
|
if (route === "ignore") continue;
|
|
7272
8306
|
const resolvedId = typeof frontmatter.id === "string" && frontmatter.id ? frontmatter.id : basenameMd(file.path);
|
|
7273
8307
|
const { body: storedBody, relatedLinks } = coreOkf.splitRelatedSection(body);
|
|
7274
|
-
const edgeLinks = isProfile1 ? relatedLinks : [...relatedLinks, ...coreOkf.extractMarkdownLinks(storedBody)];
|
|
8308
|
+
const edgeLinks = isProfile1 || isLegacyV1 ? relatedLinks : [...relatedLinks, ...coreOkf.extractMarkdownLinks(storedBody)];
|
|
8309
|
+
const usesV1StatusRule = isProfile1 || isLegacyV1 || profile === void 0 && okfVersion === void 0;
|
|
7275
8310
|
if (route === "fact") {
|
|
7276
8311
|
facts.push(frontmatterToFact(entityId, resolvedId, frontmatter, storedBody, now));
|
|
7277
8312
|
} else {
|
|
7278
|
-
tasks.push(frontmatterToTask(entityId, resolvedId, frontmatter, now));
|
|
8313
|
+
tasks.push(frontmatterToTask(entityId, resolvedId, frontmatter, now, usesV1StatusRule));
|
|
7279
8314
|
}
|
|
7280
8315
|
const seenEdges = /* @__PURE__ */ new Set();
|
|
7281
8316
|
extractEdgesFromLinks(
|
|
@@ -7384,7 +8419,9 @@ exports.PromptService = PromptService;
|
|
|
7384
8419
|
exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
7385
8420
|
exports.WikiBusyError = WikiBusyError;
|
|
7386
8421
|
exports.WikiDuplicateHashError = WikiDuplicateHashError;
|
|
8422
|
+
exports.WikiIngestEmptyError = WikiIngestEmptyError;
|
|
7387
8423
|
exports.WikiMemory = WikiMemory;
|
|
8424
|
+
exports.WikiParseError = WikiParseError;
|
|
7388
8425
|
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|
|
7389
8426
|
exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
|
|
7390
8427
|
exports.WikiTransactionError = WikiTransactionError;
|