@equationalapplications/core-llm-wiki 7.11.1 → 7.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/{chunk-IJFOAWPS.mjs → chunk-KN4XSLLB.mjs} +2103 -1414
- package/dist/chunk-KN4XSLLB.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2930 -2631
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +156 -549
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-C-WsBxLh.d.mts → testing-7d76Ptlq.d.mts} +90 -9
- package/dist/{testing-C-WsBxLh.d.ts → testing-7d76Ptlq.d.ts} +90 -9
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +431 -25
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-IJFOAWPS.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -157,6 +157,19 @@ var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
|
157
157
|
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
158
158
|
}
|
|
159
159
|
};
|
|
160
|
+
var WikiSearchUnavailableError = class extends Error {
|
|
161
|
+
constructor(message = "Keyword search is temporarily unavailable", cause) {
|
|
162
|
+
super(message, cause !== void 0 ? { cause } : void 0);
|
|
163
|
+
this.name = "WikiSearchUnavailableError";
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
function isWikiSearchUnavailableError(err) {
|
|
167
|
+
try {
|
|
168
|
+
return err instanceof WikiSearchUnavailableError;
|
|
169
|
+
} catch {
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
160
173
|
|
|
161
174
|
// src/utils/pure.ts
|
|
162
175
|
var MAX_REPAIR_CANDIDATES = 5;
|
|
@@ -655,7 +668,7 @@ function taskRejectionReason(raw) {
|
|
|
655
668
|
}
|
|
656
669
|
function normalizeSourceRef(value) {
|
|
657
670
|
if (typeof value !== "string") return null;
|
|
658
|
-
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255);
|
|
671
|
+
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255).trimEnd();
|
|
659
672
|
return cleaned.length > 0 ? cleaned : null;
|
|
660
673
|
}
|
|
661
674
|
function normalizeSourceHash(value) {
|
|
@@ -708,7 +721,7 @@ var MESSAGE = {
|
|
|
708
721
|
embedding_failed: "A fact could not be embedded.",
|
|
709
722
|
hook_failed: "A host hook threw or rejected.",
|
|
710
723
|
background_job_failed: "A background maintenance job failed.",
|
|
711
|
-
heal_skipped: "Heal skipped a candidate fact.",
|
|
724
|
+
heal_skipped: "Heal skipped a candidate fact, or the whole LLM pass when keyword search is unavailable.",
|
|
712
725
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
713
726
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
714
727
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
@@ -1709,12 +1722,24 @@ ${groundingEvidenceBlock(writer, cfg)}` : systemPrompt;
|
|
|
1709
1722
|
hasOntologyPlaceholders(template) {
|
|
1710
1723
|
return /\{\{\s*ontology(?:Manifest|ModeInstructions)\s*\}\}/.test(template);
|
|
1711
1724
|
}
|
|
1725
|
+
/** Removes ontology placeholders until none remain, so a nested form cannot reassemble one (closes CodeQL #11). */
|
|
1726
|
+
stripOntologyPlaceholders(template) {
|
|
1727
|
+
let out = template;
|
|
1728
|
+
let prev;
|
|
1729
|
+
do {
|
|
1730
|
+
prev = out;
|
|
1731
|
+
out = out.replace(/\{\{\s*ontology(?:Manifest|ModeInstructions)\s*\}\}/g, "");
|
|
1732
|
+
} while (out !== prev);
|
|
1733
|
+
return out;
|
|
1734
|
+
}
|
|
1712
1735
|
buildSystemPrompt(template, variables, ontologyContext) {
|
|
1736
|
+
const hasOntologyPlaceholders = this.hasOntologyPlaceholders(template);
|
|
1737
|
+
const effective = hasOntologyPlaceholders && ontologyContext == null ? this.stripOntologyPlaceholders(template) : template;
|
|
1713
1738
|
const shouldHydrate = Object.keys(variables).some(
|
|
1714
|
-
(key) => new RegExp(`\\{\\{\\s*${key}\\s*\\}\\}`).test(
|
|
1715
|
-
) || ontologyContext != null &&
|
|
1716
|
-
const hydrated = shouldHydrate ? this.hydrate(
|
|
1717
|
-
return
|
|
1739
|
+
(key) => new RegExp(`\\{\\{\\s*${key}\\s*\\}\\}`).test(effective)
|
|
1740
|
+
) || ontologyContext != null && hasOntologyPlaceholders;
|
|
1741
|
+
const hydrated = shouldHydrate ? this.hydrate(effective, { ...variables, ...ontologyContext ?? {} }) : effective;
|
|
1742
|
+
return hasOntologyPlaceholders ? hydrated : this.appendOntology(hydrated, ontologyContext);
|
|
1718
1743
|
}
|
|
1719
1744
|
appendOntology(systemPrompt, ctx) {
|
|
1720
1745
|
if (!ctx) return systemPrompt;
|
|
@@ -2328,6 +2353,9 @@ var IngestionService = class {
|
|
|
2328
2353
|
* ordering and counts so success counts match the pre-change contract.
|
|
2329
2354
|
*/
|
|
2330
2355
|
async upsertGraphCore(entityId, params, tx, opts) {
|
|
2356
|
+
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
2357
|
+
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
2358
|
+
params = { ...params, sourceRef };
|
|
2331
2359
|
await this.assertGraphNodeOwnership(entityId, params.nodes, tx);
|
|
2332
2360
|
const now = Date.now();
|
|
2333
2361
|
const ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
@@ -2838,6 +2866,326 @@ async function runBatched(args) {
|
|
|
2838
2866
|
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
2839
2867
|
}
|
|
2840
2868
|
|
|
2869
|
+
// src/db/schema.ts
|
|
2870
|
+
async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
2871
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}entries)`);
|
|
2872
|
+
const names = new Set(cols.map((c) => c.name));
|
|
2873
|
+
if (names.has("superseded_by")) {
|
|
2874
|
+
await db.execAsync(
|
|
2875
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
|
|
2876
|
+
);
|
|
2877
|
+
}
|
|
2878
|
+
if (names.has("valid_from") && names.has("valid_to")) {
|
|
2879
|
+
await db.execAsync(
|
|
2880
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
|
|
2881
|
+
);
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
|
|
2885
|
+
// src/db/migrations.ts
|
|
2886
|
+
var MIGRATIONS = [
|
|
2887
|
+
{
|
|
2888
|
+
version: 1,
|
|
2889
|
+
description: "Rebuild FTS5 with porter unicode61 tokenizer (superseded by v2)",
|
|
2890
|
+
run: async (_db, _prefix) => {
|
|
2891
|
+
}
|
|
2892
|
+
},
|
|
2893
|
+
{
|
|
2894
|
+
version: 2,
|
|
2895
|
+
description: "Remove FTS5; add embedding column for semantic retrieval",
|
|
2896
|
+
run: async (db, prefix) => {
|
|
2897
|
+
await db.withTransactionAsync(async (tx) => {
|
|
2898
|
+
await tx.execAsync(`
|
|
2899
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ai;
|
|
2900
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ad;
|
|
2901
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_au;
|
|
2902
|
+
DROP TABLE IF EXISTS ${prefix}entries_fts;
|
|
2903
|
+
`);
|
|
2904
|
+
});
|
|
2905
|
+
const cols = await db.getAllAsync(
|
|
2906
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
2907
|
+
);
|
|
2908
|
+
if (!cols.some((c) => c.name === "embedding")) {
|
|
2909
|
+
await db.execAsync(`ALTER TABLE ${prefix}entries ADD COLUMN embedding TEXT`);
|
|
2910
|
+
}
|
|
2911
|
+
}
|
|
2912
|
+
},
|
|
2913
|
+
{
|
|
2914
|
+
version: 3,
|
|
2915
|
+
description: "Add embedding_blob BLOB column for Float32Array vector storage",
|
|
2916
|
+
run: async (db, prefix) => {
|
|
2917
|
+
const cols = await db.getAllAsync(
|
|
2918
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
2919
|
+
);
|
|
2920
|
+
if (!cols.some((c) => c.name === "embedding_blob")) {
|
|
2921
|
+
await db.execAsync(
|
|
2922
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN embedding_blob BLOB`
|
|
2923
|
+
);
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
},
|
|
2927
|
+
{
|
|
2928
|
+
version: 4,
|
|
2929
|
+
description: "Create outbox table for change data capture",
|
|
2930
|
+
run: async (db, prefix) => {
|
|
2931
|
+
await db.execAsync(`
|
|
2932
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
2933
|
+
id TEXT PRIMARY KEY,
|
|
2934
|
+
entity_id TEXT NOT NULL,
|
|
2935
|
+
table_name TEXT NOT NULL,
|
|
2936
|
+
record_id TEXT NOT NULL,
|
|
2937
|
+
operation TEXT NOT NULL,
|
|
2938
|
+
payload TEXT NOT NULL,
|
|
2939
|
+
created_at INTEGER NOT NULL
|
|
2940
|
+
);
|
|
2941
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
2942
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
2943
|
+
`);
|
|
2944
|
+
}
|
|
2945
|
+
},
|
|
2946
|
+
{
|
|
2947
|
+
version: 5,
|
|
2948
|
+
description: "Add okf_type to entries/tasks for OKF type fidelity; create edges table for OKF graph import",
|
|
2949
|
+
run: async (db, prefix) => {
|
|
2950
|
+
for (const table of ["entries", "tasks"]) {
|
|
2951
|
+
const cols = await db.getAllAsync(
|
|
2952
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
2953
|
+
);
|
|
2954
|
+
if (!cols.some((c) => c.name === "okf_type")) {
|
|
2955
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN okf_type TEXT`);
|
|
2956
|
+
}
|
|
2957
|
+
}
|
|
2958
|
+
await db.execAsync(`
|
|
2959
|
+
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
2960
|
+
id TEXT PRIMARY KEY,
|
|
2961
|
+
entity_id TEXT NOT NULL,
|
|
2962
|
+
source_id TEXT NOT NULL,
|
|
2963
|
+
target_id TEXT NOT NULL,
|
|
2964
|
+
edge_type TEXT NOT NULL,
|
|
2965
|
+
created_at INTEGER NOT NULL,
|
|
2966
|
+
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
2967
|
+
);
|
|
2968
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_idx ON ${prefix}edges (entity_id);
|
|
2969
|
+
`);
|
|
2970
|
+
}
|
|
2971
|
+
},
|
|
2972
|
+
{
|
|
2973
|
+
version: 6,
|
|
2974
|
+
description: "Add entity_manifests table for per-entity ontology state",
|
|
2975
|
+
run: async (db, prefix) => {
|
|
2976
|
+
await db.execAsync(`
|
|
2977
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
2978
|
+
entity_id TEXT PRIMARY KEY,
|
|
2979
|
+
mode TEXT NOT NULL DEFAULT 'off',
|
|
2980
|
+
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
2981
|
+
updated_at INTEGER NOT NULL
|
|
2982
|
+
);
|
|
2983
|
+
`);
|
|
2984
|
+
}
|
|
2985
|
+
},
|
|
2986
|
+
{
|
|
2987
|
+
version: 7,
|
|
2988
|
+
description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
|
|
2989
|
+
run: async (db, prefix) => {
|
|
2990
|
+
const cols = await db.getAllAsync(
|
|
2991
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
2992
|
+
);
|
|
2993
|
+
if (!cols.some((c) => c.name === "ontology_checked_at")) {
|
|
2994
|
+
await db.execAsync(
|
|
2995
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
|
|
2996
|
+
);
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
},
|
|
3000
|
+
{
|
|
3001
|
+
version: 8,
|
|
3002
|
+
description: "Add heal_checked_at to entries for heal recheck cooldown",
|
|
3003
|
+
run: async (db, prefix) => {
|
|
3004
|
+
const cols = await db.getAllAsync(
|
|
3005
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3006
|
+
);
|
|
3007
|
+
if (!cols.some((c) => c.name === "heal_checked_at")) {
|
|
3008
|
+
await db.execAsync(
|
|
3009
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN heal_checked_at INTEGER`
|
|
3010
|
+
);
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
},
|
|
3014
|
+
{
|
|
3015
|
+
version: 9,
|
|
3016
|
+
description: "add_source_ref_index",
|
|
3017
|
+
run: async (db, prefix) => {
|
|
3018
|
+
const duplicates = await db.getAllAsync(
|
|
3019
|
+
`SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
|
|
3020
|
+
FROM ${prefix}entries
|
|
3021
|
+
WHERE deleted_at IS NULL AND source_hash IS NOT NULL
|
|
3022
|
+
GROUP BY entity_id, source_hash
|
|
3023
|
+
HAVING COUNT(DISTINCT source_ref) > 1`
|
|
3024
|
+
);
|
|
3025
|
+
if (duplicates.length > 0) {
|
|
3026
|
+
const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
|
|
3027
|
+
throw new Error(
|
|
3028
|
+
`Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
await db.execAsync(
|
|
3032
|
+
`CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
3033
|
+
id TEXT PRIMARY KEY,
|
|
3034
|
+
entity_id TEXT NOT NULL,
|
|
3035
|
+
source_hash TEXT NOT NULL,
|
|
3036
|
+
source_ref TEXT NOT NULL,
|
|
3037
|
+
created_at INTEGER NOT NULL,
|
|
3038
|
+
deleted_at INTEGER
|
|
3039
|
+
);
|
|
3040
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
3041
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
3042
|
+
WHERE deleted_at IS NULL;`
|
|
3043
|
+
);
|
|
3044
|
+
await db.execAsync(
|
|
3045
|
+
`INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
3046
|
+
SELECT
|
|
3047
|
+
'sri:' || entity_id || ':' || source_hash,
|
|
3048
|
+
entity_id,
|
|
3049
|
+
source_hash,
|
|
3050
|
+
source_ref,
|
|
3051
|
+
updated_at,
|
|
3052
|
+
NULL
|
|
3053
|
+
FROM (
|
|
3054
|
+
SELECT
|
|
3055
|
+
entity_id, source_hash, source_ref, updated_at,
|
|
3056
|
+
ROW_NUMBER() OVER (
|
|
3057
|
+
PARTITION BY entity_id, source_hash
|
|
3058
|
+
ORDER BY updated_at ASC, id ASC
|
|
3059
|
+
) AS rn
|
|
3060
|
+
FROM ${prefix}entries
|
|
3061
|
+
WHERE deleted_at IS NULL
|
|
3062
|
+
AND source_hash IS NOT NULL
|
|
3063
|
+
AND source_ref IS NOT NULL
|
|
3064
|
+
) ranked
|
|
3065
|
+
WHERE rn = 1;`
|
|
3066
|
+
);
|
|
3067
|
+
}
|
|
3068
|
+
},
|
|
3069
|
+
{
|
|
3070
|
+
version: 10,
|
|
3071
|
+
description: "OKF v0.2: add lifecycle_status, stale_after, generated_by, last_verified_*, okf_sources, okf_verified, okf_usage_window to entries and tasks",
|
|
3072
|
+
run: async (db, prefix) => {
|
|
3073
|
+
for (const table of ["entries", "tasks"]) {
|
|
3074
|
+
const cols = await db.getAllAsync(
|
|
3075
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3076
|
+
);
|
|
3077
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3078
|
+
const adds = [
|
|
3079
|
+
["lifecycle_status", `ALTER TABLE ${prefix}${table} ADD COLUMN lifecycle_status TEXT NOT NULL DEFAULT 'stable'`],
|
|
3080
|
+
["stale_after", `ALTER TABLE ${prefix}${table} ADD COLUMN stale_after INTEGER`],
|
|
3081
|
+
["generated_by", `ALTER TABLE ${prefix}${table} ADD COLUMN generated_by TEXT`],
|
|
3082
|
+
["last_verified_at", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_at INTEGER`],
|
|
3083
|
+
["last_verified_by", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_by TEXT`],
|
|
3084
|
+
["okf_sources", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_sources TEXT`],
|
|
3085
|
+
["okf_verified", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_verified TEXT`],
|
|
3086
|
+
["okf_usage_window", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_usage_window TEXT`]
|
|
3087
|
+
];
|
|
3088
|
+
for (const [name, sql] of adds) {
|
|
3089
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3090
|
+
}
|
|
3091
|
+
}
|
|
3092
|
+
await db.execAsync(`
|
|
3093
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_lifecycle_status_idx ON ${prefix}entries(lifecycle_status);
|
|
3094
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_stale_after_idx ON ${prefix}entries(stale_after);
|
|
3095
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_last_verified_at_idx ON ${prefix}entries(last_verified_at);
|
|
3096
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_lifecycle_status_idx ON ${prefix}tasks(lifecycle_status);
|
|
3097
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_stale_after_idx ON ${prefix}tasks(stale_after);
|
|
3098
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
|
|
3099
|
+
`);
|
|
3100
|
+
}
|
|
3101
|
+
},
|
|
3102
|
+
{
|
|
3103
|
+
version: 11,
|
|
3104
|
+
description: "Add embedding_failed_at, embedding_failure_kind, embedding_attempts to entries for embed retry state",
|
|
3105
|
+
run: async (db, prefix) => {
|
|
3106
|
+
const cols = await db.getAllAsync(
|
|
3107
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3108
|
+
);
|
|
3109
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3110
|
+
const adds = [
|
|
3111
|
+
["embedding_failed_at", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failed_at INTEGER`],
|
|
3112
|
+
["embedding_failure_kind", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failure_kind TEXT`],
|
|
3113
|
+
["embedding_attempts", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_attempts INTEGER NOT NULL DEFAULT 0`]
|
|
3114
|
+
];
|
|
3115
|
+
for (const [name, sql] of adds) {
|
|
3116
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
},
|
|
3120
|
+
{
|
|
3121
|
+
version: 12,
|
|
3122
|
+
description: "Replace edges_entity_idx with composite edges(entity_id, id) for keyset paging",
|
|
3123
|
+
run: async (db, prefix) => {
|
|
3124
|
+
await db.execAsync(`
|
|
3125
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
3126
|
+
DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
|
|
3127
|
+
`);
|
|
3128
|
+
}
|
|
3129
|
+
},
|
|
3130
|
+
{
|
|
3131
|
+
version: 13,
|
|
3132
|
+
description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
|
|
3133
|
+
run: async (db, prefix) => {
|
|
3134
|
+
const adds = [
|
|
3135
|
+
["entries", "valid_from", "INTEGER"],
|
|
3136
|
+
["entries", "valid_to", "INTEGER"],
|
|
3137
|
+
["entries", "superseded_by", "TEXT"],
|
|
3138
|
+
["entries", "superseded_at", "INTEGER"],
|
|
3139
|
+
["events", "occurred_at", "INTEGER"],
|
|
3140
|
+
["checkpoints", "librarian_watermark_at", "INTEGER"],
|
|
3141
|
+
["checkpoints", "librarian_watermark_id", "TEXT"]
|
|
3142
|
+
];
|
|
3143
|
+
const existingByTable = /* @__PURE__ */ new Map();
|
|
3144
|
+
for (const [table, column, type] of adds) {
|
|
3145
|
+
let existing = existingByTable.get(table);
|
|
3146
|
+
if (!existing) {
|
|
3147
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
|
|
3148
|
+
existing = new Set(cols.map((c) => c.name));
|
|
3149
|
+
existingByTable.set(table, existing);
|
|
3150
|
+
}
|
|
3151
|
+
if (!existing.has(column)) {
|
|
3152
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
|
|
3153
|
+
existing.add(column);
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
3157
|
+
}
|
|
3158
|
+
},
|
|
3159
|
+
{
|
|
3160
|
+
version: 14,
|
|
3161
|
+
description: "Partial index for the every-open legacy source_type probe",
|
|
3162
|
+
run: async (db, prefix) => {
|
|
3163
|
+
await createLegacySourceTypeIndex(db, prefix);
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
];
|
|
3167
|
+
async function createLegacySourceTypeIndex(db, prefix) {
|
|
3168
|
+
await db.execAsync(
|
|
3169
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_legacy_source_type_idx ON ${prefix}entries(source_type) WHERE source_type IN ('user_document', 'agent_inferred')`
|
|
3170
|
+
);
|
|
3171
|
+
}
|
|
3172
|
+
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
3173
|
+
if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
|
|
3174
|
+
throw new Error(
|
|
3175
|
+
`migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
|
|
3176
|
+
);
|
|
3177
|
+
}
|
|
3178
|
+
}
|
|
3179
|
+
MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
|
|
3180
|
+
|
|
3181
|
+
// src/db/sourceRefNormalize.ts
|
|
3182
|
+
var ENTRIES_WM_KEY = "normalize_v14_entries_wm";
|
|
3183
|
+
var SRI_WM_KEY = "normalize_v14_sri_wm";
|
|
3184
|
+
async function resetNormalizeWatermarks(meta) {
|
|
3185
|
+
await meta.deleteMeta(ENTRIES_WM_KEY);
|
|
3186
|
+
await meta.deleteMeta(SRI_WM_KEY);
|
|
3187
|
+
}
|
|
3188
|
+
|
|
2841
3189
|
// src/services/librarian/constants.ts
|
|
2842
3190
|
var FUZZY_THRESHOLD = 0.5;
|
|
2843
3191
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
@@ -3135,6 +3483,22 @@ function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
|
3135
3483
|
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
3136
3484
|
return { kind: "ambiguous" };
|
|
3137
3485
|
}
|
|
3486
|
+
async function loadStoredVectors(deps, entityId, nonLive) {
|
|
3487
|
+
const stored = [];
|
|
3488
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3489
|
+
if (nonLive.has(row.id)) continue;
|
|
3490
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3491
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
3492
|
+
}
|
|
3493
|
+
return stored;
|
|
3494
|
+
}
|
|
3495
|
+
async function gateNeedsKeyword(deps, entityId) {
|
|
3496
|
+
if (!await deps.entryRepo.hasLiveEntry(entityId)) return false;
|
|
3497
|
+
if (typeof deps.options.llmProvider.embed !== "function") return true;
|
|
3498
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3499
|
+
const stored = await loadStoredVectors(deps, entityId, nonLive);
|
|
3500
|
+
return stored.length === 0 || stored.some((s) => s.vec.length !== stored[0].vec.length);
|
|
3501
|
+
}
|
|
3138
3502
|
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
3139
3503
|
const { entityId, trigger } = ctx;
|
|
3140
3504
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -3150,14 +3514,8 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3150
3514
|
}
|
|
3151
3515
|
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3152
3516
|
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
3153
|
-
const stored = [];
|
|
3154
|
-
|
|
3155
|
-
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3156
|
-
if (nonLive.has(row.id)) continue;
|
|
3157
|
-
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3158
|
-
if (vec) stored.push({ id: row.id, vec });
|
|
3159
|
-
}
|
|
3160
|
-
}
|
|
3517
|
+
const stored = canEmbed && unique.length > 0 ? await loadStoredVectors(deps, entityId, nonLive) : [];
|
|
3518
|
+
const hasFacts = unique.length > 0 && await deps.entryRepo.hasLiveEntry(entityId);
|
|
3161
3519
|
const gated = [];
|
|
3162
3520
|
for (const c of unique) {
|
|
3163
3521
|
let ranked = null;
|
|
@@ -3178,7 +3536,7 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3178
3536
|
}
|
|
3179
3537
|
}
|
|
3180
3538
|
if (!ranked) {
|
|
3181
|
-
ranked = (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
3539
|
+
ranked = hasFacts ? (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null })) : [];
|
|
3182
3540
|
}
|
|
3183
3541
|
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
3184
3542
|
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
@@ -3529,6 +3887,9 @@ var runOpsLibrarianPass = async (deps, ctx) => {
|
|
|
3529
3887
|
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
3530
3888
|
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
3531
3889
|
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
3890
|
+
if (!deps.searchService.isKeywordAvailable() && await gateNeedsKeyword(deps, entityId)) {
|
|
3891
|
+
throw new WikiSearchUnavailableError("librarian: keyword search unavailable and this pass may need it; retry the pass");
|
|
3892
|
+
}
|
|
3532
3893
|
let ex;
|
|
3533
3894
|
try {
|
|
3534
3895
|
ex = await runExtract(deps, ctx, sel.batch);
|
|
@@ -3758,6 +4119,7 @@ var MaintenanceService = class {
|
|
|
3758
4119
|
}
|
|
3759
4120
|
}
|
|
3760
4121
|
if (vacuum) {
|
|
4122
|
+
await resetNormalizeWatermarks(this.metadataRepo);
|
|
3761
4123
|
await this.metadataRepo.vacuum();
|
|
3762
4124
|
}
|
|
3763
4125
|
await this.searchService.syncEntries(entityId, syncedIds);
|
|
@@ -4057,7 +4419,8 @@ var MaintenanceService = class {
|
|
|
4057
4419
|
*
|
|
4058
4420
|
* Bounded: at most `batchSize` candidates per pass (#67). Loop on
|
|
4059
4421
|
* `result.remaining > 0` for convergence — see {@link HealResult.remaining}
|
|
4060
|
-
* for what convergence means here
|
|
4422
|
+
* for what convergence means here, and for the no-progress flags
|
|
4423
|
+
* (`searchUnavailable`, `budgetStop`) a loop must also stop on.
|
|
4061
4424
|
*/
|
|
4062
4425
|
async doRunHeal(entityId, options) {
|
|
4063
4426
|
const promptOverride = options?.promptOverride;
|
|
@@ -4103,7 +4466,11 @@ var MaintenanceService = class {
|
|
|
4103
4466
|
}
|
|
4104
4467
|
}
|
|
4105
4468
|
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId, batchSize, recheckCutoff);
|
|
4106
|
-
|
|
4469
|
+
const keywordUnavailable = healCandidates.length > 0 && !this.searchService.isKeywordAvailable();
|
|
4470
|
+
if (healCandidates.length === 0 || keywordUnavailable) {
|
|
4471
|
+
if (keywordUnavailable) {
|
|
4472
|
+
emitDiagnostic(this.options, { ...diagBase, code: "heal_skipped", detail: { reason: "keyword_search_unavailable", count: healCandidates.length } });
|
|
4473
|
+
}
|
|
4107
4474
|
await this.searchService.syncEntries(entityId, Array.from(/* @__PURE__ */ new Set([...orphanedIds, ...staleDowngradedIds])));
|
|
4108
4475
|
this.searchService.evictCache(entityId);
|
|
4109
4476
|
const counts2 = await this.entryRepo.countHealCandidatesByEntityId(entityId, recheckCutoff);
|
|
@@ -4115,7 +4482,10 @@ var MaintenanceService = class {
|
|
|
4115
4482
|
skipped: [],
|
|
4116
4483
|
degraded: [],
|
|
4117
4484
|
remaining: counts2.eligible,
|
|
4118
|
-
deferred: counts2.deferred
|
|
4485
|
+
deferred: counts2.deferred,
|
|
4486
|
+
// remaining stays > 0 so maybeRunHeal holds the checkpoint back and
|
|
4487
|
+
// retries after the outage; the flag lets a host loop stop instead.
|
|
4488
|
+
...keywordUnavailable ? { searchUnavailable: true } : {}
|
|
4119
4489
|
};
|
|
4120
4490
|
}
|
|
4121
4491
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
@@ -4919,7 +5289,12 @@ var RetrievalService = class {
|
|
|
4919
5289
|
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
4920
5290
|
}
|
|
4921
5291
|
if (weight !== void 0 && weight < 1) {
|
|
4922
|
-
|
|
5292
|
+
try {
|
|
5293
|
+
miniSearchScores = await this.searchService.getKeywordScores(trimmedQuery, scoredEntityIds);
|
|
5294
|
+
} catch (err) {
|
|
5295
|
+
if (!isWikiSearchUnavailableError(err)) throw err;
|
|
5296
|
+
miniSearchScores = /* @__PURE__ */ new Map();
|
|
5297
|
+
}
|
|
4923
5298
|
}
|
|
4924
5299
|
}
|
|
4925
5300
|
if (candidateRows === null) {
|
|
@@ -5421,6 +5796,28 @@ function toIndexDoc(row) {
|
|
|
5421
5796
|
};
|
|
5422
5797
|
}
|
|
5423
5798
|
|
|
5799
|
+
// src/services/search/UnavailableIndexStrategy.ts
|
|
5800
|
+
var UnavailableIndexStrategy = class {
|
|
5801
|
+
constructor(cause) {
|
|
5802
|
+
this.cause = cause;
|
|
5803
|
+
}
|
|
5804
|
+
async search(_query, options) {
|
|
5805
|
+
if (options.entityIds.length === 0) return [];
|
|
5806
|
+
throw new WikiSearchUnavailableError(void 0, this.cause);
|
|
5807
|
+
}
|
|
5808
|
+
async replace(_entityId, _ids, _documents) {
|
|
5809
|
+
}
|
|
5810
|
+
async replaceEntity(_entityId, _documents) {
|
|
5811
|
+
}
|
|
5812
|
+
async replaceAll(_documents) {
|
|
5813
|
+
}
|
|
5814
|
+
hasIndexedEntity(_entityId) {
|
|
5815
|
+
return false;
|
|
5816
|
+
}
|
|
5817
|
+
async drain() {
|
|
5818
|
+
}
|
|
5819
|
+
};
|
|
5820
|
+
|
|
5424
5821
|
// src/services/SearchService.ts
|
|
5425
5822
|
var _SearchService = class _SearchService {
|
|
5426
5823
|
constructor(entryRepo, indexStrategy) {
|
|
@@ -5459,6 +5856,10 @@ var _SearchService = class _SearchService {
|
|
|
5459
5856
|
setIndexStrategy(strategy) {
|
|
5460
5857
|
this.indexStrategy = strategy;
|
|
5461
5858
|
}
|
|
5859
|
+
/** False while the installed strategy is the degraded UnavailableIndexStrategy (spec #280 Change 4). */
|
|
5860
|
+
isKeywordAvailable() {
|
|
5861
|
+
return !(this.indexStrategy instanceof UnavailableIndexStrategy);
|
|
5862
|
+
}
|
|
5462
5863
|
/**
|
|
5463
5864
|
* One serialized drain turn for strategies with a durable ledger. Like
|
|
5464
5865
|
* sync(), it never rejects: the keyword index is rebuildable, and a failed
|
|
@@ -6173,6 +6574,7 @@ var WriteService = class {
|
|
|
6173
6574
|
this.jobManager = jobManager;
|
|
6174
6575
|
this.maintenanceService = maintenanceService;
|
|
6175
6576
|
this.inflight = /* @__PURE__ */ new Set();
|
|
6577
|
+
this.warnedSearchUnavailable = false;
|
|
6176
6578
|
}
|
|
6177
6579
|
track(p) {
|
|
6178
6580
|
this.inflight.add(p);
|
|
@@ -6269,8 +6671,7 @@ var WriteService = class {
|
|
|
6269
6671
|
this.jobManager.acquireLock("librarian", entityId);
|
|
6270
6672
|
const stage = { job: "librarian" };
|
|
6271
6673
|
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
6272
|
-
|
|
6273
|
-
this.reportBackgroundFailure(entityId, stage.job);
|
|
6674
|
+
this.reportBackgroundFailure(entityId, stage.job, err);
|
|
6274
6675
|
}).finally(() => {
|
|
6275
6676
|
this.jobManager.releaseLock("librarian", entityId);
|
|
6276
6677
|
});
|
|
@@ -6281,8 +6682,7 @@ var WriteService = class {
|
|
|
6281
6682
|
}
|
|
6282
6683
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
6283
6684
|
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
6284
|
-
|
|
6285
|
-
this.reportBackgroundFailure(entityId, "heal");
|
|
6685
|
+
this.reportBackgroundFailure(entityId, "heal", err);
|
|
6286
6686
|
});
|
|
6287
6687
|
this.track(healJob);
|
|
6288
6688
|
}
|
|
@@ -6320,13 +6720,19 @@ var WriteService = class {
|
|
|
6320
6720
|
}
|
|
6321
6721
|
}
|
|
6322
6722
|
}
|
|
6323
|
-
reportBackgroundFailure(entityId, job) {
|
|
6723
|
+
reportBackgroundFailure(entityId, job, err) {
|
|
6724
|
+
const searchUnavailable = isWikiSearchUnavailableError(err);
|
|
6725
|
+
if (!searchUnavailable) console.error(err);
|
|
6726
|
+
else if (!this.warnedSearchUnavailable) {
|
|
6727
|
+
this.warnedSearchUnavailable = true;
|
|
6728
|
+
console.warn("[WikiMemory] Auto librarian/heal pass skipped: keyword search is unavailable. Work is held, not lost, until it recovers (a no-arg syncSearchIndex() retries the init); later skips report only via onDiagnostic.");
|
|
6729
|
+
}
|
|
6324
6730
|
emitDiagnostic(this.options, {
|
|
6325
6731
|
code: "background_job_failed",
|
|
6326
6732
|
operation: job,
|
|
6327
6733
|
trigger: "auto",
|
|
6328
6734
|
entityId,
|
|
6329
|
-
detail: { reason: "unhandled_rejection" }
|
|
6735
|
+
detail: { reason: searchUnavailable ? "keyword_search_unavailable" : "unhandled_rejection" }
|
|
6330
6736
|
});
|
|
6331
6737
|
}
|
|
6332
6738
|
};
|