@equationalapplications/core-llm-wiki 7.11.2 → 7.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/{chunk-522SCN2E.mjs → chunk-KN4XSLLB.mjs} +2088 -1411
- package/dist/chunk-KN4XSLLB.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +4775 -4488
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +155 -548
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-B0UtFCq1.d.mts → testing-7d76Ptlq.d.mts} +88 -9
- package/dist/{testing-B0UtFCq1.d.ts → testing-7d76Ptlq.d.ts} +88 -9
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +415 -21
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-522SCN2E.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -157,6 +157,19 @@ var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
|
157
157
|
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
158
158
|
}
|
|
159
159
|
};
|
|
160
|
+
var WikiSearchUnavailableError = class extends Error {
|
|
161
|
+
constructor(message = "Keyword search is temporarily unavailable", cause) {
|
|
162
|
+
super(message, cause !== void 0 ? { cause } : void 0);
|
|
163
|
+
this.name = "WikiSearchUnavailableError";
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
function isWikiSearchUnavailableError(err) {
|
|
167
|
+
try {
|
|
168
|
+
return err instanceof WikiSearchUnavailableError;
|
|
169
|
+
} catch {
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
160
173
|
|
|
161
174
|
// src/utils/pure.ts
|
|
162
175
|
var MAX_REPAIR_CANDIDATES = 5;
|
|
@@ -655,7 +668,7 @@ function taskRejectionReason(raw) {
|
|
|
655
668
|
}
|
|
656
669
|
function normalizeSourceRef(value) {
|
|
657
670
|
if (typeof value !== "string") return null;
|
|
658
|
-
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255);
|
|
671
|
+
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255).trimEnd();
|
|
659
672
|
return cleaned.length > 0 ? cleaned : null;
|
|
660
673
|
}
|
|
661
674
|
function normalizeSourceHash(value) {
|
|
@@ -708,7 +721,7 @@ var MESSAGE = {
|
|
|
708
721
|
embedding_failed: "A fact could not be embedded.",
|
|
709
722
|
hook_failed: "A host hook threw or rejected.",
|
|
710
723
|
background_job_failed: "A background maintenance job failed.",
|
|
711
|
-
heal_skipped: "Heal skipped a candidate fact.",
|
|
724
|
+
heal_skipped: "Heal skipped a candidate fact, or the whole LLM pass when keyword search is unavailable.",
|
|
712
725
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
713
726
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
714
727
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
@@ -2340,6 +2353,9 @@ var IngestionService = class {
|
|
|
2340
2353
|
* ordering and counts so success counts match the pre-change contract.
|
|
2341
2354
|
*/
|
|
2342
2355
|
async upsertGraphCore(entityId, params, tx, opts) {
|
|
2356
|
+
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
2357
|
+
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
2358
|
+
params = { ...params, sourceRef };
|
|
2343
2359
|
await this.assertGraphNodeOwnership(entityId, params.nodes, tx);
|
|
2344
2360
|
const now = Date.now();
|
|
2345
2361
|
const ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
@@ -2850,6 +2866,326 @@ async function runBatched(args) {
|
|
|
2850
2866
|
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
2851
2867
|
}
|
|
2852
2868
|
|
|
2869
|
+
// src/db/schema.ts
|
|
2870
|
+
async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
2871
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}entries)`);
|
|
2872
|
+
const names = new Set(cols.map((c) => c.name));
|
|
2873
|
+
if (names.has("superseded_by")) {
|
|
2874
|
+
await db.execAsync(
|
|
2875
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
|
|
2876
|
+
);
|
|
2877
|
+
}
|
|
2878
|
+
if (names.has("valid_from") && names.has("valid_to")) {
|
|
2879
|
+
await db.execAsync(
|
|
2880
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
|
|
2881
|
+
);
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
|
|
2885
|
+
// src/db/migrations.ts
|
|
2886
|
+
var MIGRATIONS = [
|
|
2887
|
+
{
|
|
2888
|
+
version: 1,
|
|
2889
|
+
description: "Rebuild FTS5 with porter unicode61 tokenizer (superseded by v2)",
|
|
2890
|
+
run: async (_db, _prefix) => {
|
|
2891
|
+
}
|
|
2892
|
+
},
|
|
2893
|
+
{
|
|
2894
|
+
version: 2,
|
|
2895
|
+
description: "Remove FTS5; add embedding column for semantic retrieval",
|
|
2896
|
+
run: async (db, prefix) => {
|
|
2897
|
+
await db.withTransactionAsync(async (tx) => {
|
|
2898
|
+
await tx.execAsync(`
|
|
2899
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ai;
|
|
2900
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ad;
|
|
2901
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_au;
|
|
2902
|
+
DROP TABLE IF EXISTS ${prefix}entries_fts;
|
|
2903
|
+
`);
|
|
2904
|
+
});
|
|
2905
|
+
const cols = await db.getAllAsync(
|
|
2906
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
2907
|
+
);
|
|
2908
|
+
if (!cols.some((c) => c.name === "embedding")) {
|
|
2909
|
+
await db.execAsync(`ALTER TABLE ${prefix}entries ADD COLUMN embedding TEXT`);
|
|
2910
|
+
}
|
|
2911
|
+
}
|
|
2912
|
+
},
|
|
2913
|
+
{
|
|
2914
|
+
version: 3,
|
|
2915
|
+
description: "Add embedding_blob BLOB column for Float32Array vector storage",
|
|
2916
|
+
run: async (db, prefix) => {
|
|
2917
|
+
const cols = await db.getAllAsync(
|
|
2918
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
2919
|
+
);
|
|
2920
|
+
if (!cols.some((c) => c.name === "embedding_blob")) {
|
|
2921
|
+
await db.execAsync(
|
|
2922
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN embedding_blob BLOB`
|
|
2923
|
+
);
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
},
|
|
2927
|
+
{
|
|
2928
|
+
version: 4,
|
|
2929
|
+
description: "Create outbox table for change data capture",
|
|
2930
|
+
run: async (db, prefix) => {
|
|
2931
|
+
await db.execAsync(`
|
|
2932
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
2933
|
+
id TEXT PRIMARY KEY,
|
|
2934
|
+
entity_id TEXT NOT NULL,
|
|
2935
|
+
table_name TEXT NOT NULL,
|
|
2936
|
+
record_id TEXT NOT NULL,
|
|
2937
|
+
operation TEXT NOT NULL,
|
|
2938
|
+
payload TEXT NOT NULL,
|
|
2939
|
+
created_at INTEGER NOT NULL
|
|
2940
|
+
);
|
|
2941
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
2942
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
2943
|
+
`);
|
|
2944
|
+
}
|
|
2945
|
+
},
|
|
2946
|
+
{
|
|
2947
|
+
version: 5,
|
|
2948
|
+
description: "Add okf_type to entries/tasks for OKF type fidelity; create edges table for OKF graph import",
|
|
2949
|
+
run: async (db, prefix) => {
|
|
2950
|
+
for (const table of ["entries", "tasks"]) {
|
|
2951
|
+
const cols = await db.getAllAsync(
|
|
2952
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
2953
|
+
);
|
|
2954
|
+
if (!cols.some((c) => c.name === "okf_type")) {
|
|
2955
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN okf_type TEXT`);
|
|
2956
|
+
}
|
|
2957
|
+
}
|
|
2958
|
+
await db.execAsync(`
|
|
2959
|
+
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
2960
|
+
id TEXT PRIMARY KEY,
|
|
2961
|
+
entity_id TEXT NOT NULL,
|
|
2962
|
+
source_id TEXT NOT NULL,
|
|
2963
|
+
target_id TEXT NOT NULL,
|
|
2964
|
+
edge_type TEXT NOT NULL,
|
|
2965
|
+
created_at INTEGER NOT NULL,
|
|
2966
|
+
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
2967
|
+
);
|
|
2968
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_idx ON ${prefix}edges (entity_id);
|
|
2969
|
+
`);
|
|
2970
|
+
}
|
|
2971
|
+
},
|
|
2972
|
+
{
|
|
2973
|
+
version: 6,
|
|
2974
|
+
description: "Add entity_manifests table for per-entity ontology state",
|
|
2975
|
+
run: async (db, prefix) => {
|
|
2976
|
+
await db.execAsync(`
|
|
2977
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
2978
|
+
entity_id TEXT PRIMARY KEY,
|
|
2979
|
+
mode TEXT NOT NULL DEFAULT 'off',
|
|
2980
|
+
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
2981
|
+
updated_at INTEGER NOT NULL
|
|
2982
|
+
);
|
|
2983
|
+
`);
|
|
2984
|
+
}
|
|
2985
|
+
},
|
|
2986
|
+
{
|
|
2987
|
+
version: 7,
|
|
2988
|
+
description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
|
|
2989
|
+
run: async (db, prefix) => {
|
|
2990
|
+
const cols = await db.getAllAsync(
|
|
2991
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
2992
|
+
);
|
|
2993
|
+
if (!cols.some((c) => c.name === "ontology_checked_at")) {
|
|
2994
|
+
await db.execAsync(
|
|
2995
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
|
|
2996
|
+
);
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
},
|
|
3000
|
+
{
|
|
3001
|
+
version: 8,
|
|
3002
|
+
description: "Add heal_checked_at to entries for heal recheck cooldown",
|
|
3003
|
+
run: async (db, prefix) => {
|
|
3004
|
+
const cols = await db.getAllAsync(
|
|
3005
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3006
|
+
);
|
|
3007
|
+
if (!cols.some((c) => c.name === "heal_checked_at")) {
|
|
3008
|
+
await db.execAsync(
|
|
3009
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN heal_checked_at INTEGER`
|
|
3010
|
+
);
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
},
|
|
3014
|
+
{
|
|
3015
|
+
version: 9,
|
|
3016
|
+
description: "add_source_ref_index",
|
|
3017
|
+
run: async (db, prefix) => {
|
|
3018
|
+
const duplicates = await db.getAllAsync(
|
|
3019
|
+
`SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
|
|
3020
|
+
FROM ${prefix}entries
|
|
3021
|
+
WHERE deleted_at IS NULL AND source_hash IS NOT NULL
|
|
3022
|
+
GROUP BY entity_id, source_hash
|
|
3023
|
+
HAVING COUNT(DISTINCT source_ref) > 1`
|
|
3024
|
+
);
|
|
3025
|
+
if (duplicates.length > 0) {
|
|
3026
|
+
const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
|
|
3027
|
+
throw new Error(
|
|
3028
|
+
`Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
await db.execAsync(
|
|
3032
|
+
`CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
3033
|
+
id TEXT PRIMARY KEY,
|
|
3034
|
+
entity_id TEXT NOT NULL,
|
|
3035
|
+
source_hash TEXT NOT NULL,
|
|
3036
|
+
source_ref TEXT NOT NULL,
|
|
3037
|
+
created_at INTEGER NOT NULL,
|
|
3038
|
+
deleted_at INTEGER
|
|
3039
|
+
);
|
|
3040
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
3041
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
3042
|
+
WHERE deleted_at IS NULL;`
|
|
3043
|
+
);
|
|
3044
|
+
await db.execAsync(
|
|
3045
|
+
`INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
3046
|
+
SELECT
|
|
3047
|
+
'sri:' || entity_id || ':' || source_hash,
|
|
3048
|
+
entity_id,
|
|
3049
|
+
source_hash,
|
|
3050
|
+
source_ref,
|
|
3051
|
+
updated_at,
|
|
3052
|
+
NULL
|
|
3053
|
+
FROM (
|
|
3054
|
+
SELECT
|
|
3055
|
+
entity_id, source_hash, source_ref, updated_at,
|
|
3056
|
+
ROW_NUMBER() OVER (
|
|
3057
|
+
PARTITION BY entity_id, source_hash
|
|
3058
|
+
ORDER BY updated_at ASC, id ASC
|
|
3059
|
+
) AS rn
|
|
3060
|
+
FROM ${prefix}entries
|
|
3061
|
+
WHERE deleted_at IS NULL
|
|
3062
|
+
AND source_hash IS NOT NULL
|
|
3063
|
+
AND source_ref IS NOT NULL
|
|
3064
|
+
) ranked
|
|
3065
|
+
WHERE rn = 1;`
|
|
3066
|
+
);
|
|
3067
|
+
}
|
|
3068
|
+
},
|
|
3069
|
+
{
|
|
3070
|
+
version: 10,
|
|
3071
|
+
description: "OKF v0.2: add lifecycle_status, stale_after, generated_by, last_verified_*, okf_sources, okf_verified, okf_usage_window to entries and tasks",
|
|
3072
|
+
run: async (db, prefix) => {
|
|
3073
|
+
for (const table of ["entries", "tasks"]) {
|
|
3074
|
+
const cols = await db.getAllAsync(
|
|
3075
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3076
|
+
);
|
|
3077
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3078
|
+
const adds = [
|
|
3079
|
+
["lifecycle_status", `ALTER TABLE ${prefix}${table} ADD COLUMN lifecycle_status TEXT NOT NULL DEFAULT 'stable'`],
|
|
3080
|
+
["stale_after", `ALTER TABLE ${prefix}${table} ADD COLUMN stale_after INTEGER`],
|
|
3081
|
+
["generated_by", `ALTER TABLE ${prefix}${table} ADD COLUMN generated_by TEXT`],
|
|
3082
|
+
["last_verified_at", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_at INTEGER`],
|
|
3083
|
+
["last_verified_by", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_by TEXT`],
|
|
3084
|
+
["okf_sources", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_sources TEXT`],
|
|
3085
|
+
["okf_verified", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_verified TEXT`],
|
|
3086
|
+
["okf_usage_window", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_usage_window TEXT`]
|
|
3087
|
+
];
|
|
3088
|
+
for (const [name, sql] of adds) {
|
|
3089
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3090
|
+
}
|
|
3091
|
+
}
|
|
3092
|
+
await db.execAsync(`
|
|
3093
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_lifecycle_status_idx ON ${prefix}entries(lifecycle_status);
|
|
3094
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_stale_after_idx ON ${prefix}entries(stale_after);
|
|
3095
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_last_verified_at_idx ON ${prefix}entries(last_verified_at);
|
|
3096
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_lifecycle_status_idx ON ${prefix}tasks(lifecycle_status);
|
|
3097
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_stale_after_idx ON ${prefix}tasks(stale_after);
|
|
3098
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
|
|
3099
|
+
`);
|
|
3100
|
+
}
|
|
3101
|
+
},
|
|
3102
|
+
{
|
|
3103
|
+
version: 11,
|
|
3104
|
+
description: "Add embedding_failed_at, embedding_failure_kind, embedding_attempts to entries for embed retry state",
|
|
3105
|
+
run: async (db, prefix) => {
|
|
3106
|
+
const cols = await db.getAllAsync(
|
|
3107
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3108
|
+
);
|
|
3109
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3110
|
+
const adds = [
|
|
3111
|
+
["embedding_failed_at", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failed_at INTEGER`],
|
|
3112
|
+
["embedding_failure_kind", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failure_kind TEXT`],
|
|
3113
|
+
["embedding_attempts", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_attempts INTEGER NOT NULL DEFAULT 0`]
|
|
3114
|
+
];
|
|
3115
|
+
for (const [name, sql] of adds) {
|
|
3116
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
},
|
|
3120
|
+
{
|
|
3121
|
+
version: 12,
|
|
3122
|
+
description: "Replace edges_entity_idx with composite edges(entity_id, id) for keyset paging",
|
|
3123
|
+
run: async (db, prefix) => {
|
|
3124
|
+
await db.execAsync(`
|
|
3125
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
3126
|
+
DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
|
|
3127
|
+
`);
|
|
3128
|
+
}
|
|
3129
|
+
},
|
|
3130
|
+
{
|
|
3131
|
+
version: 13,
|
|
3132
|
+
description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
|
|
3133
|
+
run: async (db, prefix) => {
|
|
3134
|
+
const adds = [
|
|
3135
|
+
["entries", "valid_from", "INTEGER"],
|
|
3136
|
+
["entries", "valid_to", "INTEGER"],
|
|
3137
|
+
["entries", "superseded_by", "TEXT"],
|
|
3138
|
+
["entries", "superseded_at", "INTEGER"],
|
|
3139
|
+
["events", "occurred_at", "INTEGER"],
|
|
3140
|
+
["checkpoints", "librarian_watermark_at", "INTEGER"],
|
|
3141
|
+
["checkpoints", "librarian_watermark_id", "TEXT"]
|
|
3142
|
+
];
|
|
3143
|
+
const existingByTable = /* @__PURE__ */ new Map();
|
|
3144
|
+
for (const [table, column, type] of adds) {
|
|
3145
|
+
let existing = existingByTable.get(table);
|
|
3146
|
+
if (!existing) {
|
|
3147
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
|
|
3148
|
+
existing = new Set(cols.map((c) => c.name));
|
|
3149
|
+
existingByTable.set(table, existing);
|
|
3150
|
+
}
|
|
3151
|
+
if (!existing.has(column)) {
|
|
3152
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
|
|
3153
|
+
existing.add(column);
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
3157
|
+
}
|
|
3158
|
+
},
|
|
3159
|
+
{
|
|
3160
|
+
version: 14,
|
|
3161
|
+
description: "Partial index for the every-open legacy source_type probe",
|
|
3162
|
+
run: async (db, prefix) => {
|
|
3163
|
+
await createLegacySourceTypeIndex(db, prefix);
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
];
|
|
3167
|
+
async function createLegacySourceTypeIndex(db, prefix) {
|
|
3168
|
+
await db.execAsync(
|
|
3169
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_legacy_source_type_idx ON ${prefix}entries(source_type) WHERE source_type IN ('user_document', 'agent_inferred')`
|
|
3170
|
+
);
|
|
3171
|
+
}
|
|
3172
|
+
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
3173
|
+
if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
|
|
3174
|
+
throw new Error(
|
|
3175
|
+
`migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
|
|
3176
|
+
);
|
|
3177
|
+
}
|
|
3178
|
+
}
|
|
3179
|
+
MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
|
|
3180
|
+
|
|
3181
|
+
// src/db/sourceRefNormalize.ts
|
|
3182
|
+
var ENTRIES_WM_KEY = "normalize_v14_entries_wm";
|
|
3183
|
+
var SRI_WM_KEY = "normalize_v14_sri_wm";
|
|
3184
|
+
async function resetNormalizeWatermarks(meta) {
|
|
3185
|
+
await meta.deleteMeta(ENTRIES_WM_KEY);
|
|
3186
|
+
await meta.deleteMeta(SRI_WM_KEY);
|
|
3187
|
+
}
|
|
3188
|
+
|
|
2853
3189
|
// src/services/librarian/constants.ts
|
|
2854
3190
|
var FUZZY_THRESHOLD = 0.5;
|
|
2855
3191
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
@@ -3147,6 +3483,22 @@ function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
|
3147
3483
|
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
3148
3484
|
return { kind: "ambiguous" };
|
|
3149
3485
|
}
|
|
3486
|
+
async function loadStoredVectors(deps, entityId, nonLive) {
|
|
3487
|
+
const stored = [];
|
|
3488
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3489
|
+
if (nonLive.has(row.id)) continue;
|
|
3490
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3491
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
3492
|
+
}
|
|
3493
|
+
return stored;
|
|
3494
|
+
}
|
|
3495
|
+
async function gateNeedsKeyword(deps, entityId) {
|
|
3496
|
+
if (!await deps.entryRepo.hasLiveEntry(entityId)) return false;
|
|
3497
|
+
if (typeof deps.options.llmProvider.embed !== "function") return true;
|
|
3498
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3499
|
+
const stored = await loadStoredVectors(deps, entityId, nonLive);
|
|
3500
|
+
return stored.length === 0 || stored.some((s) => s.vec.length !== stored[0].vec.length);
|
|
3501
|
+
}
|
|
3150
3502
|
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
3151
3503
|
const { entityId, trigger } = ctx;
|
|
3152
3504
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -3162,14 +3514,8 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3162
3514
|
}
|
|
3163
3515
|
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3164
3516
|
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
3165
|
-
const stored = [];
|
|
3166
|
-
|
|
3167
|
-
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3168
|
-
if (nonLive.has(row.id)) continue;
|
|
3169
|
-
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3170
|
-
if (vec) stored.push({ id: row.id, vec });
|
|
3171
|
-
}
|
|
3172
|
-
}
|
|
3517
|
+
const stored = canEmbed && unique.length > 0 ? await loadStoredVectors(deps, entityId, nonLive) : [];
|
|
3518
|
+
const hasFacts = unique.length > 0 && await deps.entryRepo.hasLiveEntry(entityId);
|
|
3173
3519
|
const gated = [];
|
|
3174
3520
|
for (const c of unique) {
|
|
3175
3521
|
let ranked = null;
|
|
@@ -3190,7 +3536,7 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3190
3536
|
}
|
|
3191
3537
|
}
|
|
3192
3538
|
if (!ranked) {
|
|
3193
|
-
ranked = (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
3539
|
+
ranked = hasFacts ? (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null })) : [];
|
|
3194
3540
|
}
|
|
3195
3541
|
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
3196
3542
|
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
@@ -3541,6 +3887,9 @@ var runOpsLibrarianPass = async (deps, ctx) => {
|
|
|
3541
3887
|
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
3542
3888
|
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
3543
3889
|
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
3890
|
+
if (!deps.searchService.isKeywordAvailable() && await gateNeedsKeyword(deps, entityId)) {
|
|
3891
|
+
throw new WikiSearchUnavailableError("librarian: keyword search unavailable and this pass may need it; retry the pass");
|
|
3892
|
+
}
|
|
3544
3893
|
let ex;
|
|
3545
3894
|
try {
|
|
3546
3895
|
ex = await runExtract(deps, ctx, sel.batch);
|
|
@@ -3770,6 +4119,7 @@ var MaintenanceService = class {
|
|
|
3770
4119
|
}
|
|
3771
4120
|
}
|
|
3772
4121
|
if (vacuum) {
|
|
4122
|
+
await resetNormalizeWatermarks(this.metadataRepo);
|
|
3773
4123
|
await this.metadataRepo.vacuum();
|
|
3774
4124
|
}
|
|
3775
4125
|
await this.searchService.syncEntries(entityId, syncedIds);
|
|
@@ -4069,7 +4419,8 @@ var MaintenanceService = class {
|
|
|
4069
4419
|
*
|
|
4070
4420
|
* Bounded: at most `batchSize` candidates per pass (#67). Loop on
|
|
4071
4421
|
* `result.remaining > 0` for convergence — see {@link HealResult.remaining}
|
|
4072
|
-
* for what convergence means here
|
|
4422
|
+
* for what convergence means here, and for the no-progress flags
|
|
4423
|
+
* (`searchUnavailable`, `budgetStop`) a loop must also stop on.
|
|
4073
4424
|
*/
|
|
4074
4425
|
async doRunHeal(entityId, options) {
|
|
4075
4426
|
const promptOverride = options?.promptOverride;
|
|
@@ -4115,7 +4466,11 @@ var MaintenanceService = class {
|
|
|
4115
4466
|
}
|
|
4116
4467
|
}
|
|
4117
4468
|
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId, batchSize, recheckCutoff);
|
|
4118
|
-
|
|
4469
|
+
const keywordUnavailable = healCandidates.length > 0 && !this.searchService.isKeywordAvailable();
|
|
4470
|
+
if (healCandidates.length === 0 || keywordUnavailable) {
|
|
4471
|
+
if (keywordUnavailable) {
|
|
4472
|
+
emitDiagnostic(this.options, { ...diagBase, code: "heal_skipped", detail: { reason: "keyword_search_unavailable", count: healCandidates.length } });
|
|
4473
|
+
}
|
|
4119
4474
|
await this.searchService.syncEntries(entityId, Array.from(/* @__PURE__ */ new Set([...orphanedIds, ...staleDowngradedIds])));
|
|
4120
4475
|
this.searchService.evictCache(entityId);
|
|
4121
4476
|
const counts2 = await this.entryRepo.countHealCandidatesByEntityId(entityId, recheckCutoff);
|
|
@@ -4127,7 +4482,10 @@ var MaintenanceService = class {
|
|
|
4127
4482
|
skipped: [],
|
|
4128
4483
|
degraded: [],
|
|
4129
4484
|
remaining: counts2.eligible,
|
|
4130
|
-
deferred: counts2.deferred
|
|
4485
|
+
deferred: counts2.deferred,
|
|
4486
|
+
// remaining stays > 0 so maybeRunHeal holds the checkpoint back and
|
|
4487
|
+
// retries after the outage; the flag lets a host loop stop instead.
|
|
4488
|
+
...keywordUnavailable ? { searchUnavailable: true } : {}
|
|
4131
4489
|
};
|
|
4132
4490
|
}
|
|
4133
4491
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
@@ -4931,7 +5289,12 @@ var RetrievalService = class {
|
|
|
4931
5289
|
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
4932
5290
|
}
|
|
4933
5291
|
if (weight !== void 0 && weight < 1) {
|
|
4934
|
-
|
|
5292
|
+
try {
|
|
5293
|
+
miniSearchScores = await this.searchService.getKeywordScores(trimmedQuery, scoredEntityIds);
|
|
5294
|
+
} catch (err) {
|
|
5295
|
+
if (!isWikiSearchUnavailableError(err)) throw err;
|
|
5296
|
+
miniSearchScores = /* @__PURE__ */ new Map();
|
|
5297
|
+
}
|
|
4935
5298
|
}
|
|
4936
5299
|
}
|
|
4937
5300
|
if (candidateRows === null) {
|
|
@@ -5433,6 +5796,28 @@ function toIndexDoc(row) {
|
|
|
5433
5796
|
};
|
|
5434
5797
|
}
|
|
5435
5798
|
|
|
5799
|
+
// src/services/search/UnavailableIndexStrategy.ts
|
|
5800
|
+
var UnavailableIndexStrategy = class {
|
|
5801
|
+
constructor(cause) {
|
|
5802
|
+
this.cause = cause;
|
|
5803
|
+
}
|
|
5804
|
+
async search(_query, options) {
|
|
5805
|
+
if (options.entityIds.length === 0) return [];
|
|
5806
|
+
throw new WikiSearchUnavailableError(void 0, this.cause);
|
|
5807
|
+
}
|
|
5808
|
+
async replace(_entityId, _ids, _documents) {
|
|
5809
|
+
}
|
|
5810
|
+
async replaceEntity(_entityId, _documents) {
|
|
5811
|
+
}
|
|
5812
|
+
async replaceAll(_documents) {
|
|
5813
|
+
}
|
|
5814
|
+
hasIndexedEntity(_entityId) {
|
|
5815
|
+
return false;
|
|
5816
|
+
}
|
|
5817
|
+
async drain() {
|
|
5818
|
+
}
|
|
5819
|
+
};
|
|
5820
|
+
|
|
5436
5821
|
// src/services/SearchService.ts
|
|
5437
5822
|
var _SearchService = class _SearchService {
|
|
5438
5823
|
constructor(entryRepo, indexStrategy) {
|
|
@@ -5471,6 +5856,10 @@ var _SearchService = class _SearchService {
|
|
|
5471
5856
|
setIndexStrategy(strategy) {
|
|
5472
5857
|
this.indexStrategy = strategy;
|
|
5473
5858
|
}
|
|
5859
|
+
/** False while the installed strategy is the degraded UnavailableIndexStrategy (spec #280 Change 4). */
|
|
5860
|
+
isKeywordAvailable() {
|
|
5861
|
+
return !(this.indexStrategy instanceof UnavailableIndexStrategy);
|
|
5862
|
+
}
|
|
5474
5863
|
/**
|
|
5475
5864
|
* One serialized drain turn for strategies with a durable ledger. Like
|
|
5476
5865
|
* sync(), it never rejects: the keyword index is rebuildable, and a failed
|
|
@@ -6185,6 +6574,7 @@ var WriteService = class {
|
|
|
6185
6574
|
this.jobManager = jobManager;
|
|
6186
6575
|
this.maintenanceService = maintenanceService;
|
|
6187
6576
|
this.inflight = /* @__PURE__ */ new Set();
|
|
6577
|
+
this.warnedSearchUnavailable = false;
|
|
6188
6578
|
}
|
|
6189
6579
|
track(p) {
|
|
6190
6580
|
this.inflight.add(p);
|
|
@@ -6281,8 +6671,7 @@ var WriteService = class {
|
|
|
6281
6671
|
this.jobManager.acquireLock("librarian", entityId);
|
|
6282
6672
|
const stage = { job: "librarian" };
|
|
6283
6673
|
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
6284
|
-
|
|
6285
|
-
this.reportBackgroundFailure(entityId, stage.job);
|
|
6674
|
+
this.reportBackgroundFailure(entityId, stage.job, err);
|
|
6286
6675
|
}).finally(() => {
|
|
6287
6676
|
this.jobManager.releaseLock("librarian", entityId);
|
|
6288
6677
|
});
|
|
@@ -6293,8 +6682,7 @@ var WriteService = class {
|
|
|
6293
6682
|
}
|
|
6294
6683
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
6295
6684
|
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
6296
|
-
|
|
6297
|
-
this.reportBackgroundFailure(entityId, "heal");
|
|
6685
|
+
this.reportBackgroundFailure(entityId, "heal", err);
|
|
6298
6686
|
});
|
|
6299
6687
|
this.track(healJob);
|
|
6300
6688
|
}
|
|
@@ -6332,13 +6720,19 @@ var WriteService = class {
|
|
|
6332
6720
|
}
|
|
6333
6721
|
}
|
|
6334
6722
|
}
|
|
6335
|
-
reportBackgroundFailure(entityId, job) {
|
|
6723
|
+
reportBackgroundFailure(entityId, job, err) {
|
|
6724
|
+
const searchUnavailable = isWikiSearchUnavailableError(err);
|
|
6725
|
+
if (!searchUnavailable) console.error(err);
|
|
6726
|
+
else if (!this.warnedSearchUnavailable) {
|
|
6727
|
+
this.warnedSearchUnavailable = true;
|
|
6728
|
+
console.warn("[WikiMemory] Auto librarian/heal pass skipped: keyword search is unavailable. Work is held, not lost, until it recovers (a no-arg syncSearchIndex() retries the init); later skips report only via onDiagnostic.");
|
|
6729
|
+
}
|
|
6336
6730
|
emitDiagnostic(this.options, {
|
|
6337
6731
|
code: "background_job_failed",
|
|
6338
6732
|
operation: job,
|
|
6339
6733
|
trigger: "auto",
|
|
6340
6734
|
entityId,
|
|
6341
|
-
detail: { reason: "unhandled_rejection" }
|
|
6735
|
+
detail: { reason: searchUnavailable ? "keyword_search_unavailable" : "unhandled_rejection" }
|
|
6342
6736
|
});
|
|
6343
6737
|
}
|
|
6344
6738
|
};
|