@equationalapplications/core-llm-wiki 7.3.0 → 7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -0
- package/dist/{chunk-IAYO63DQ.mjs → chunk-3P7FAKJA.mjs} +167 -18
- package/dist/chunk-3P7FAKJA.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +252 -17
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +89 -4
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-ht1qOvcD.d.mts → testing-ClpOUiLI.d.mts} +139 -2
- package/dist/{testing-ht1qOvcD.d.ts → testing-ClpOUiLI.d.ts} +139 -2
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +156 -15
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-IAYO63DQ.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, D as DegradedRecord, E as EmbedFactResult,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-ClpOUiLI.mjs';
|
|
2
|
+
export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityStatus, l as ExtractedFact, m as ExtractedFactEdge, n as ExtractedFactWithOntology, o as ExtractedTask, p as GraphTraversalOptions, H as HEAL_BATCH_SIZE, q as HEAL_RECHECK_MS, r as HOOK_TIMEOUT_MARKER, s as HealResult, I as IngestDocumentResult, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, t as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, u as ONTOLOGY_BACKFILL_RECHECK_MS, v as OntologyBackfillResult, w as OntologyConfig, x as OntologyEdgeType, y as OntologyManifest, z as OntologyMode, A as OntologyNodeType, B as OntologyPromptContext, J as OntologyUpdates, P as PromptOverrides, K as PromptService, N as PrunePartialFailureError, Q as ReembedResult, V as VectorRanker, T as VectorRankerFallback, U as VectorRankerRankArgs, X as VectorRankerSemanticResult, Y as WikiBusyError, Z as WikiBusyOperation, _ as WikiCheckpoint, $ as WikiConfig, a0 as WikiDiagnostic, a1 as WikiDiagnosticCode, a2 as WikiDiagnosticDetail, a3 as WikiDiagnosticOperation, a4 as WikiDiagnosticSeverity, a5 as WikiDiagnosticTrigger, a6 as WikiDraftNotFound, a7 as WikiDuplicateHashError, a8 as WikiEdge, a9 as WikiEvent, aa as WikiFact, ab as WikiGraphNodeOwnershipConflict, ac as WikiIngestEmptyError, ad as WikiInvalidReadOptions, ae as WikiMemoryTestAccess, af as WikiOutboxEvent, ag as WikiParseError, ah as WikiSourceRefHashCollision, ai as WikiStrictOntologyViolation, aj as WikiTask, ak as WikiTransactionError, al as validateManifest } from './testing-ClpOUiLI.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, D as DegradedRecord, E as EmbedFactResult,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-ClpOUiLI.js';
|
|
2
|
+
export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityStatus, l as ExtractedFact, m as ExtractedFactEdge, n as ExtractedFactWithOntology, o as ExtractedTask, p as GraphTraversalOptions, H as HEAL_BATCH_SIZE, q as HEAL_RECHECK_MS, r as HOOK_TIMEOUT_MARKER, s as HealResult, I as IngestDocumentResult, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, t as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, u as ONTOLOGY_BACKFILL_RECHECK_MS, v as OntologyBackfillResult, w as OntologyConfig, x as OntologyEdgeType, y as OntologyManifest, z as OntologyMode, A as OntologyNodeType, B as OntologyPromptContext, J as OntologyUpdates, P as PromptOverrides, K as PromptService, N as PrunePartialFailureError, Q as ReembedResult, V as VectorRanker, T as VectorRankerFallback, U as VectorRankerRankArgs, X as VectorRankerSemanticResult, Y as WikiBusyError, Z as WikiBusyOperation, _ as WikiCheckpoint, $ as WikiConfig, a0 as WikiDiagnostic, a1 as WikiDiagnosticCode, a2 as WikiDiagnosticDetail, a3 as WikiDiagnosticOperation, a4 as WikiDiagnosticSeverity, a5 as WikiDiagnosticTrigger, a6 as WikiDraftNotFound, a7 as WikiDuplicateHashError, a8 as WikiEdge, a9 as WikiEvent, aa as WikiFact, ab as WikiGraphNodeOwnershipConflict, ac as WikiIngestEmptyError, ad as WikiInvalidReadOptions, ae as WikiMemoryTestAccess, af as WikiOutboxEvent, ag as WikiParseError, ah as WikiSourceRefHashCollision, ai as WikiStrictOntologyViolation, aj as WikiTask, ak as WikiTransactionError, al as validateManifest } from './testing-ClpOUiLI.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
package/dist/index.js
CHANGED
|
@@ -333,6 +333,14 @@ var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
|
|
|
333
333
|
Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
|
|
334
334
|
}
|
|
335
335
|
};
|
|
336
|
+
var WikiDraftNotFound = class _WikiDraftNotFound extends Error {
|
|
337
|
+
constructor() {
|
|
338
|
+
super("No draft fact with that id exists for this entity.");
|
|
339
|
+
this.code = "WIKI_DRAFT_NOT_FOUND";
|
|
340
|
+
this.name = "WikiDraftNotFound";
|
|
341
|
+
Object.setPrototypeOf(this, _WikiDraftNotFound.prototype);
|
|
342
|
+
}
|
|
343
|
+
};
|
|
336
344
|
|
|
337
345
|
// src/db/serializedAdapter.ts
|
|
338
346
|
function isDriverError(err) {
|
|
@@ -1757,16 +1765,52 @@ var EntryRepository = class extends BaseRepository {
|
|
|
1757
1765
|
`UPDATE ${this.prefix}entries SET source_type = 'librarian_inferred' WHERE source_type = 'agent_inferred';`
|
|
1758
1766
|
].join("\n");
|
|
1759
1767
|
}
|
|
1760
|
-
async findRecentByEntityIds(entityIds, limit, tx) {
|
|
1768
|
+
async findRecentByEntityIds(entityIds, limit, tx, opts) {
|
|
1761
1769
|
if (entityIds.length === 0) return [];
|
|
1762
1770
|
const executor = this.getExecutor(tx);
|
|
1763
1771
|
const placeholders = entityIds.map(() => "?").join(",");
|
|
1772
|
+
const draftClause = opts?.excludeDrafts === true ? ` AND lifecycle_status != 'draft'` : "";
|
|
1764
1773
|
const rows = await executor.getAllAsync(
|
|
1765
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
1774
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL${draftClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
1766
1775
|
[...entityIds, limit]
|
|
1767
1776
|
);
|
|
1768
1777
|
return rows.map(mapRowToFact);
|
|
1769
1778
|
}
|
|
1779
|
+
/** IDs of live draft facts for the given entities (spec §5.1). Callers pass ≤ 100 entity ids. */
|
|
1780
|
+
async findDraftIdsByEntityIds(entityIds, tx) {
|
|
1781
|
+
if (entityIds.length === 0) return /* @__PURE__ */ new Set();
|
|
1782
|
+
const executor = this.getExecutor(tx);
|
|
1783
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
1784
|
+
const rows = await executor.getAllAsync(
|
|
1785
|
+
`SELECT id FROM ${this.prefix}entries
|
|
1786
|
+
WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL AND lifecycle_status = 'draft'`,
|
|
1787
|
+
[...entityIds]
|
|
1788
|
+
);
|
|
1789
|
+
return new Set(rows.map((r) => r.id));
|
|
1790
|
+
}
|
|
1791
|
+
/** Live drafts for one entity, newest first, keyset-paged by (created_at, id). */
|
|
1792
|
+
async listDraftsByEntityId(entityId, limit, after, tx) {
|
|
1793
|
+
const executor = this.getExecutor(tx);
|
|
1794
|
+
const afterClause = after ? ` AND (created_at < ? OR (created_at = ? AND id < ?))` : "";
|
|
1795
|
+
const args = after ? [entityId, after.createdAt, after.createdAt, after.id, limit] : [entityId, limit];
|
|
1796
|
+
const rows = await executor.getAllAsync(
|
|
1797
|
+
`SELECT * FROM ${this.prefix}entries
|
|
1798
|
+
WHERE entity_id = ? AND deleted_at IS NULL AND lifecycle_status = 'draft'${afterClause}
|
|
1799
|
+
ORDER BY created_at DESC, id DESC LIMIT ?`,
|
|
1800
|
+
args
|
|
1801
|
+
);
|
|
1802
|
+
return rows.map(mapRowToFact);
|
|
1803
|
+
}
|
|
1804
|
+
/** True iff `entryId` is a live draft owned by `entityId`. */
|
|
1805
|
+
async isLiveDraft(entryId, entityId, tx) {
|
|
1806
|
+
const executor = this.getExecutor(tx);
|
|
1807
|
+
const row = await executor.getFirstAsync(
|
|
1808
|
+
`SELECT id FROM ${this.prefix}entries
|
|
1809
|
+
WHERE id = ? AND entity_id = ? AND deleted_at IS NULL AND lifecycle_status = 'draft'`,
|
|
1810
|
+
[entryId, entityId]
|
|
1811
|
+
);
|
|
1812
|
+
return row != null;
|
|
1813
|
+
}
|
|
1770
1814
|
/**
|
|
1771
1815
|
* Live untyped facts eligible for ontology backfill, oldest first.
|
|
1772
1816
|
* Skips facts checked within the recheck cooldown (ontology_checked_at > recheckCutoff).
|
|
@@ -2577,6 +2621,7 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
2577
2621
|
END
|
|
2578
2622
|
) >= ?
|
|
2579
2623
|
AND n.source_type NOT IN (${excludeSourceTypesPlaceholders})
|
|
2624
|
+
AND (? = 0 OR n.lifecycle_status != 'draft')
|
|
2580
2625
|
WHERE w.depth < ?
|
|
2581
2626
|
AND instr(w.visited, ',' || (CASE WHEN e.source_id = w.node_id THEN e.target_id ELSE e.source_id END) || ',') = 0
|
|
2582
2627
|
)
|
|
@@ -2596,6 +2641,7 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
2596
2641
|
entityId,
|
|
2597
2642
|
minConfidenceRank,
|
|
2598
2643
|
...opts.excludeSourceTypes,
|
|
2644
|
+
opts.excludeDrafts === true ? 1 : 0,
|
|
2599
2645
|
opts.maxDepth,
|
|
2600
2646
|
opts.maxNodes
|
|
2601
2647
|
];
|
|
@@ -5066,6 +5112,51 @@ var IngestionService = class {
|
|
|
5066
5112
|
}
|
|
5067
5113
|
};
|
|
5068
5114
|
|
|
5115
|
+
// src/utils/classifier.ts
|
|
5116
|
+
var isUnit = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
|
|
5117
|
+
function validateClassifierAnswer(response, key, question) {
|
|
5118
|
+
try {
|
|
5119
|
+
if (response === null || typeof response !== "object") return { ok: false, reason: "malformed" };
|
|
5120
|
+
const answers = response.answers;
|
|
5121
|
+
if (answers === null || typeof answers !== "object") return { ok: false, reason: "missing_answer" };
|
|
5122
|
+
if (!Object.prototype.hasOwnProperty.call(answers, key)) return { ok: false, reason: "missing_answer" };
|
|
5123
|
+
const raw = answers[key];
|
|
5124
|
+
if (raw === null || typeof raw !== "object") return { ok: false, reason: "missing_answer" };
|
|
5125
|
+
const a = raw;
|
|
5126
|
+
if (a.kind !== question.kind) return { ok: false, reason: "kind_mismatch" };
|
|
5127
|
+
if (question.kind === "choice") {
|
|
5128
|
+
if (typeof a.choice !== "string" || !question.options.includes(a.choice)) return { ok: false, reason: "choice_not_offered" };
|
|
5129
|
+
if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
|
|
5130
|
+
const probs = a.probabilities;
|
|
5131
|
+
if (probs === null || typeof probs !== "object" || Array.isArray(probs)) return { ok: false, reason: "invalid_probability" };
|
|
5132
|
+
const probabilities = {};
|
|
5133
|
+
for (const [k, v] of Object.entries(probs)) {
|
|
5134
|
+
if (!isUnit(v)) return { ok: false, reason: "invalid_probability" };
|
|
5135
|
+
probabilities[k] = v;
|
|
5136
|
+
}
|
|
5137
|
+
return { ok: true, answer: { kind: "choice", choice: a.choice, confidence: a.confidence, probabilities } };
|
|
5138
|
+
}
|
|
5139
|
+
if (question.kind === "binary") {
|
|
5140
|
+
if (!isUnit(a.probability)) return { ok: false, reason: "invalid_probability" };
|
|
5141
|
+
return { ok: true, answer: { kind: "binary", probability: a.probability } };
|
|
5142
|
+
}
|
|
5143
|
+
const maxScore = question.levels.length - 1;
|
|
5144
|
+
if (typeof a.score !== "number" || !Number.isFinite(a.score) || a.score < 0 || a.score > maxScore) {
|
|
5145
|
+
return { ok: false, reason: "score_out_of_range" };
|
|
5146
|
+
}
|
|
5147
|
+
if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
|
|
5148
|
+
if (!Array.isArray(a.probabilities) || !a.probabilities.every(isUnit)) return { ok: false, reason: "invalid_probability" };
|
|
5149
|
+
return { ok: true, answer: { kind: "score", score: a.score, confidence: a.confidence, probabilities: a.probabilities.slice() } };
|
|
5150
|
+
} catch {
|
|
5151
|
+
return { ok: false, reason: "malformed" };
|
|
5152
|
+
}
|
|
5153
|
+
}
|
|
5154
|
+
function classifierStateForFact(fact) {
|
|
5155
|
+
const parts = [fact.title, fact.body];
|
|
5156
|
+
if (Array.isArray(fact.tags) && fact.tags.length > 0) parts.push(`Tags: ${fact.tags.join(", ")}`);
|
|
5157
|
+
return parts.join("\n\n");
|
|
5158
|
+
}
|
|
5159
|
+
|
|
5069
5160
|
// src/services/BoundedLlmCall.ts
|
|
5070
5161
|
var DEFAULT_BATCH_SIZE = 10;
|
|
5071
5162
|
var ESTIMATED_OUTPUT_TOKENS_PER_ITEM = 150;
|
|
@@ -5948,7 +6039,7 @@ var MaintenanceService = class {
|
|
|
5948
6039
|
if (!ontologyService) {
|
|
5949
6040
|
return { ...zeroed, remaining: 0, deferred: 0 };
|
|
5950
6041
|
}
|
|
5951
|
-
const { mode } = await ontologyService.getEffectiveState(entityId);
|
|
6042
|
+
const { mode, manifest: effectiveManifest } = await ontologyService.getEffectiveState(entityId);
|
|
5952
6043
|
if (mode === "off") {
|
|
5953
6044
|
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
5954
6045
|
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
@@ -5958,6 +6049,14 @@ var MaintenanceService = class {
|
|
|
5958
6049
|
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
5959
6050
|
return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
|
|
5960
6051
|
}
|
|
6052
|
+
const classifierMode = options?.classifier ?? this.options.config?.ontology?.backfillClassifier ?? "llm";
|
|
6053
|
+
const classify = this.options.llmProvider.classify;
|
|
6054
|
+
if (classifierMode === "auto" && typeof classify === "function") {
|
|
6055
|
+
const slugs = effectiveManifest.node_types.map((n) => n.type);
|
|
6056
|
+
if (slugs.length > 0 && slugs.length <= 255) {
|
|
6057
|
+
return this._runClassifierBackfill(entityId, candidates, effectiveManifest, now, recheckCutoff);
|
|
6058
|
+
}
|
|
6059
|
+
}
|
|
5961
6060
|
const ontologyContext = await ontologyService.buildPromptContext(entityId);
|
|
5962
6061
|
const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
|
|
5963
6062
|
const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
|
|
@@ -6036,6 +6135,87 @@ var MaintenanceService = class {
|
|
|
6036
6135
|
deferred: counts.deferred
|
|
6037
6136
|
};
|
|
6038
6137
|
}
|
|
6138
|
+
/**
|
|
6139
|
+
* Classifier-mode backfill (spec §7.3): one `choice` question per untyped
|
|
6140
|
+
* fact over the effective manifest's node-type slugs. Accepted answers go
|
|
6141
|
+
* through `_applyOntologyBackfillBatch` exactly like LLM classifications.
|
|
6142
|
+
* No edges are proposed.
|
|
6143
|
+
*/
|
|
6144
|
+
async _runClassifierBackfill(entityId, candidates, manifest, now, recheckCutoff) {
|
|
6145
|
+
const provider = this.options.llmProvider;
|
|
6146
|
+
const rawMin = this.options.config?.ontology?.classifyMinConfidence;
|
|
6147
|
+
const minConfidence = typeof rawMin === "number" && Number.isFinite(rawMin) && rawMin >= 0 && rawMin <= 1 ? rawMin : 0.5;
|
|
6148
|
+
const rawConcurrency = this.options.config?.chunkConcurrency ?? 1;
|
|
6149
|
+
const concurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
|
|
6150
|
+
const question = {
|
|
6151
|
+
kind: "choice",
|
|
6152
|
+
options: manifest.node_types.map((n) => n.type),
|
|
6153
|
+
instructions: "Choose the ontology node type that best describes this fact.\n" + manifest.node_types.map((n) => `- ${n.type}: ${n.description}`).join("\n")
|
|
6154
|
+
};
|
|
6155
|
+
const outcomes = await withConcurrency(
|
|
6156
|
+
candidates.map((fact) => async () => {
|
|
6157
|
+
let response;
|
|
6158
|
+
try {
|
|
6159
|
+
response = await provider.classify({ state: classifierStateForFact(fact), questions: { okf_type: question } });
|
|
6160
|
+
} catch {
|
|
6161
|
+
return { fact, kind: "threw" };
|
|
6162
|
+
}
|
|
6163
|
+
const checked = validateClassifierAnswer(response, "okf_type", question);
|
|
6164
|
+
if (!checked.ok) return { fact, kind: "invalid", reason: checked.reason };
|
|
6165
|
+
if (checked.answer.kind !== "choice") return { fact, kind: "invalid", reason: "kind_mismatch" };
|
|
6166
|
+
if (checked.answer.confidence < minConfidence) return { fact, kind: "low_confidence" };
|
|
6167
|
+
return { fact, kind: "accepted", okfType: checked.answer.choice };
|
|
6168
|
+
}),
|
|
6169
|
+
concurrency
|
|
6170
|
+
);
|
|
6171
|
+
const attempted = outcomes.filter((o) => o.kind !== "threw").map((o) => o.fact);
|
|
6172
|
+
const skipped = outcomes.length - attempted.length;
|
|
6173
|
+
const classifications = outcomes.filter((o) => o.kind === "accepted").map((o) => ({ id: o.fact.id, okf_type: o.okfType }));
|
|
6174
|
+
const invalidCount = outcomes.filter((o) => o.kind === "invalid").length;
|
|
6175
|
+
let typed = 0;
|
|
6176
|
+
let failedValidation = 0;
|
|
6177
|
+
let edgesAdded = 0;
|
|
6178
|
+
let aborted = false;
|
|
6179
|
+
if (attempted.length > 0) {
|
|
6180
|
+
const applied = await this._applyOntologyBackfillBatch(
|
|
6181
|
+
entityId,
|
|
6182
|
+
{ batch: attempted, classifications, ontologyUpdates: void 0 },
|
|
6183
|
+
now
|
|
6184
|
+
);
|
|
6185
|
+
aborted = applied.abortedOntologyOff;
|
|
6186
|
+
if (!aborted) {
|
|
6187
|
+
typed = applied.typed;
|
|
6188
|
+
failedValidation = invalidCount + applied.failedValidation;
|
|
6189
|
+
edgesAdded = applied.edgesAdded;
|
|
6190
|
+
}
|
|
6191
|
+
}
|
|
6192
|
+
const diagBuffer = new DiagnosticBuffer();
|
|
6193
|
+
const diagBase = { entityId, operation: "ontologyBackfill", trigger: "call" };
|
|
6194
|
+
for (const o of outcomes) {
|
|
6195
|
+
if (o.kind === "low_confidence") {
|
|
6196
|
+
diagBuffer.push({ ...diagBase, code: "classification_low_confidence", detail: { factId: o.fact.id, reason: "below_threshold" } });
|
|
6197
|
+
} else if (o.kind === "invalid") {
|
|
6198
|
+
diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: o.reason } });
|
|
6199
|
+
} else if (o.kind === "threw") {
|
|
6200
|
+
diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: "classify_threw" } });
|
|
6201
|
+
}
|
|
6202
|
+
}
|
|
6203
|
+
diagBuffer.flush(this.options);
|
|
6204
|
+
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
6205
|
+
if (aborted) {
|
|
6206
|
+
return { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0, skipped, remaining: 0, deferred: counts.deferred };
|
|
6207
|
+
}
|
|
6208
|
+
this.searchService.evictCache(entityId);
|
|
6209
|
+
return {
|
|
6210
|
+
scanned: candidates.length,
|
|
6211
|
+
typed,
|
|
6212
|
+
failedValidation,
|
|
6213
|
+
edgesAdded,
|
|
6214
|
+
skipped,
|
|
6215
|
+
remaining: counts.eligible,
|
|
6216
|
+
deferred: counts.deferred
|
|
6217
|
+
};
|
|
6218
|
+
}
|
|
6039
6219
|
/**
|
|
6040
6220
|
* Applies one parsed backfill batch in its own transaction. Per-batch rather
|
|
6041
6221
|
* than one transaction for the pass, so mergeEmergentUpdates semantics and
|
|
@@ -6739,8 +6919,7 @@ var EmbeddingService = class {
|
|
|
6739
6919
|
}
|
|
6740
6920
|
}
|
|
6741
6921
|
async tryEmbedFact(fact, ctx) {
|
|
6742
|
-
|
|
6743
|
-
if (typeof embedFn !== "function") return { ok: false, kind: "no_provider" };
|
|
6922
|
+
if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
|
|
6744
6923
|
let tagsStr;
|
|
6745
6924
|
if (Array.isArray(fact.tags)) {
|
|
6746
6925
|
tagsStr = fact.tags.join(" ");
|
|
@@ -6757,7 +6936,7 @@ var EmbeddingService = class {
|
|
|
6757
6936
|
const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
|
|
6758
6937
|
let float32Vector;
|
|
6759
6938
|
try {
|
|
6760
|
-
const vector = await
|
|
6939
|
+
const vector = await this.options.llmProvider.embed(text);
|
|
6761
6940
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
6762
6941
|
console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
|
|
6763
6942
|
this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
@@ -6969,6 +7148,7 @@ function selectWithFloors(sortedRows, floors, maxResults) {
|
|
|
6969
7148
|
}
|
|
6970
7149
|
|
|
6971
7150
|
// src/services/RetrievalService.ts
|
|
7151
|
+
var EMPTY_ID_SET = /* @__PURE__ */ new Set();
|
|
6972
7152
|
var RetrievalService = class {
|
|
6973
7153
|
constructor(options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService) {
|
|
6974
7154
|
this.options = options;
|
|
@@ -7001,6 +7181,7 @@ var RetrievalService = class {
|
|
|
7001
7181
|
}
|
|
7002
7182
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
7003
7183
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
7184
|
+
const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
|
|
7004
7185
|
const trimmedQuery = query.trim();
|
|
7005
7186
|
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
7006
7187
|
entityIds,
|
|
@@ -7014,19 +7195,21 @@ var RetrievalService = class {
|
|
|
7014
7195
|
const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
|
|
7015
7196
|
const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
|
|
7016
7197
|
const skipEmbed = weight === 0;
|
|
7017
|
-
const embedFn = this.options.llmProvider.embed;
|
|
7018
7198
|
let facts = [];
|
|
7019
7199
|
let scoreByFactId;
|
|
7020
7200
|
if (maxResults === 0) ; else if (trimmedQuery) {
|
|
7021
7201
|
let usedEmbed = false;
|
|
7022
7202
|
const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
|
|
7203
|
+
const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
|
|
7204
|
+
const draftPad = draftIds.size;
|
|
7205
|
+
const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + draftPad;
|
|
7023
7206
|
if (scoredEntityIds.length === 0) {
|
|
7024
7207
|
usedEmbed = true;
|
|
7025
|
-
} else if (!skipEmbed &&
|
|
7208
|
+
} else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
|
|
7026
7209
|
let rankerShouldRethrow = false;
|
|
7027
7210
|
let pendingRankerFallbackError;
|
|
7028
7211
|
try {
|
|
7029
|
-
const queryVec = await
|
|
7212
|
+
const queryVec = await this.options.llmProvider.embed(trimmedQuery);
|
|
7030
7213
|
if (queryVec.length === 0 || !queryVec.every((v) => typeof v === "number" && isFinite(v))) {
|
|
7031
7214
|
throw new Error(
|
|
7032
7215
|
"embed() returned an empty or non-finite vector. Falling back to keyword search."
|
|
@@ -7053,7 +7236,7 @@ var RetrievalService = class {
|
|
|
7053
7236
|
let miniSearchScores;
|
|
7054
7237
|
if (effectivePreFilterLimit !== void 0) {
|
|
7055
7238
|
populateCache = false;
|
|
7056
|
-
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER);
|
|
7239
|
+
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !draftIds.has(r.id));
|
|
7057
7240
|
if (preResults.length === 0) {
|
|
7058
7241
|
candidateRows = null;
|
|
7059
7242
|
} else {
|
|
@@ -7075,9 +7258,9 @@ var RetrievalService = class {
|
|
|
7075
7258
|
}
|
|
7076
7259
|
} else {
|
|
7077
7260
|
if (useRanker) {
|
|
7078
|
-
candidateRows = await this.entryRepo.findMetadataByEntityIds(scoredEntityIds);
|
|
7261
|
+
candidateRows = this._withoutDrafts(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), draftIds);
|
|
7079
7262
|
} else {
|
|
7080
|
-
candidateRows = await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds);
|
|
7263
|
+
candidateRows = this._withoutDrafts(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), draftIds);
|
|
7081
7264
|
}
|
|
7082
7265
|
if (weight !== void 0 && weight < 1) {
|
|
7083
7266
|
miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
|
|
@@ -7107,7 +7290,7 @@ var RetrievalService = class {
|
|
|
7107
7290
|
candidateRows: rowsForEntity,
|
|
7108
7291
|
weight,
|
|
7109
7292
|
miniSearchScores,
|
|
7110
|
-
limit: Math.max(maxResults * 2, maxResults + 50)
|
|
7293
|
+
limit: padLimit(Math.max(maxResults * 2, maxResults + 50))
|
|
7111
7294
|
});
|
|
7112
7295
|
return ranked.map((row) => ({ ...row, entity_id: scopedEntityId }));
|
|
7113
7296
|
})
|
|
@@ -7321,7 +7504,7 @@ var RetrievalService = class {
|
|
|
7321
7504
|
});
|
|
7322
7505
|
} else if (policy === "keyword") {
|
|
7323
7506
|
const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
7324
|
-
const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
7507
|
+
const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
|
|
7325
7508
|
const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
|
|
7326
7509
|
const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
|
|
7327
7510
|
const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
|
|
@@ -7362,6 +7545,7 @@ var RetrievalService = class {
|
|
|
7362
7545
|
// read() re-sorts after applying tier weights
|
|
7363
7546
|
});
|
|
7364
7547
|
}
|
|
7548
|
+
if (draftPad > 0) scored = scored.filter((s) => !draftIds.has(s.id));
|
|
7365
7549
|
if (scored.length > 0) {
|
|
7366
7550
|
scored = scored.map((row) => ({
|
|
7367
7551
|
...row,
|
|
@@ -7424,8 +7608,8 @@ var RetrievalService = class {
|
|
|
7424
7608
|
}
|
|
7425
7609
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
7426
7610
|
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
7427
|
-
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
7428
|
-
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit);
|
|
7611
|
+
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
|
|
7612
|
+
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !draftIds.has(r.id));
|
|
7429
7613
|
const candidates = results.map((r) => ({
|
|
7430
7614
|
id: r.id,
|
|
7431
7615
|
entity_id: r.entity_id,
|
|
@@ -7449,7 +7633,7 @@ var RetrievalService = class {
|
|
|
7449
7633
|
await this.entryRepo.trackAccess(ids, now);
|
|
7450
7634
|
}
|
|
7451
7635
|
} else {
|
|
7452
|
-
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults);
|
|
7636
|
+
facts = excludeDrafts ? await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, { excludeDrafts: true }) : await this.entryRepo.findRecentByEntityIds(entityIds, maxResults);
|
|
7453
7637
|
}
|
|
7454
7638
|
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
7455
7639
|
const [tasks, events] = await Promise.all([
|
|
@@ -7485,6 +7669,9 @@ var RetrievalService = class {
|
|
|
7485
7669
|
_tieBreakSort(items) {
|
|
7486
7670
|
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
7487
7671
|
}
|
|
7672
|
+
_withoutDrafts(rows, draftIds) {
|
|
7673
|
+
return draftIds.size === 0 ? rows : rows.filter((row) => !draftIds.has(row.id));
|
|
7674
|
+
}
|
|
7488
7675
|
/**
|
|
7489
7676
|
* Comparator for score + deterministic tie-break fields.
|
|
7490
7677
|
* Negative return means "a ranks ahead of b" for descending score order.
|
|
@@ -7881,6 +8068,7 @@ var GraphTraversalService = class {
|
|
|
7881
8068
|
edgeTypes: options.edgeTypes,
|
|
7882
8069
|
minConfidence: options.minTraversalConfidence ?? this.config.minTraversalConfidence ?? "tentative",
|
|
7883
8070
|
excludeSourceTypes: options.excludeSourceTypes ?? this.config.excludeSourceTypes ?? [],
|
|
8071
|
+
excludeDrafts: options.excludeDrafts ?? this.config.excludeDrafts ?? false,
|
|
7884
8072
|
maxNodes
|
|
7885
8073
|
};
|
|
7886
8074
|
const { nodeIds, edges } = await this.edgeRepo.getNeighborhood(entityId, options.sourceId, opts);
|
|
@@ -8054,6 +8242,15 @@ var OkfTrustWritesRepository = class {
|
|
|
8054
8242
|
|
|
8055
8243
|
// src/WikiMemory.ts
|
|
8056
8244
|
var TABLE_PREFIX_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,30}_$/;
|
|
8245
|
+
function decodeDraftCursor(cursor) {
|
|
8246
|
+
const sep = typeof cursor === "string" ? cursor.indexOf(":") : -1;
|
|
8247
|
+
const createdAt = sep > 0 ? Number(cursor.slice(0, sep)) : Number.NaN;
|
|
8248
|
+
const id = sep > 0 ? cursor.slice(sep + 1) : "";
|
|
8249
|
+
if (!Number.isSafeInteger(createdAt) || id.length === 0) {
|
|
8250
|
+
throw new TypeError("Invalid listDrafts cursor.");
|
|
8251
|
+
}
|
|
8252
|
+
return { createdAt, id };
|
|
8253
|
+
}
|
|
8057
8254
|
var _testAccessNonTestEnvWarned;
|
|
8058
8255
|
var WikiMemory = class {
|
|
8059
8256
|
constructor(db, options) {
|
|
@@ -8598,6 +8795,43 @@ var WikiMemory = class {
|
|
|
8598
8795
|
async setLifecycleStatus(entryId, entityId, status) {
|
|
8599
8796
|
return this.okfTrustWrites.setLifecycleStatus(entryId, entityId, status);
|
|
8600
8797
|
}
|
|
8798
|
+
/**
|
|
8799
|
+
* Live draft facts for one entity, newest first (spec §5.2). Default page
|
|
8800
|
+
* size 50, clamped to [1, 500]. Pass `nextCursor` back unchanged for the next page.
|
|
8801
|
+
*/
|
|
8802
|
+
async listDrafts(entityId, options) {
|
|
8803
|
+
const rawLimit = options?.limit;
|
|
8804
|
+
const limit = typeof rawLimit === "number" && Number.isFinite(rawLimit) ? Math.min(500, Math.max(1, Math.floor(rawLimit))) : 50;
|
|
8805
|
+
const after = options?.cursor === void 0 ? null : decodeDraftCursor(options.cursor);
|
|
8806
|
+
const rows = await this.entryRepo.listDraftsByEntityId(entityId, limit + 1, after);
|
|
8807
|
+
const facts = rows.slice(0, limit);
|
|
8808
|
+
const last = facts[facts.length - 1];
|
|
8809
|
+
return {
|
|
8810
|
+
facts,
|
|
8811
|
+
nextCursor: rows.length > limit && last ? `${last.created_at}:${last.id}` : null
|
|
8812
|
+
};
|
|
8813
|
+
}
|
|
8814
|
+
/**
|
|
8815
|
+
* Promote a draft to `stable` and record who reviewed it, atomically
|
|
8816
|
+
* (spec §5.2). Metadata writes only: `updated_at` is not bumped and no
|
|
8817
|
+
* outbox event is pushed. Pass `by: 'human:<id>'` so `trustTier` becomes
|
|
8818
|
+
* `'human-reviewed'`.
|
|
8819
|
+
*
|
|
8820
|
+
* @throws WikiDraftNotFound when no live draft with that id exists for the entity.
|
|
8821
|
+
*/
|
|
8822
|
+
async promoteDraft(entryId, entityId, reviewer) {
|
|
8823
|
+
const by = reviewer?.by;
|
|
8824
|
+
if (typeof by !== "string" || by.trim().length === 0) {
|
|
8825
|
+
throw new TypeError("promoteDraft requires reviewer.by to be a non-empty string.");
|
|
8826
|
+
}
|
|
8827
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
8828
|
+
if (!await this.entryRepo.isLiveDraft(entryId, entityId, tx)) {
|
|
8829
|
+
throw new WikiDraftNotFound();
|
|
8830
|
+
}
|
|
8831
|
+
await this.okfTrustWrites.setLifecycleStatus(entryId, entityId, "stable", tx);
|
|
8832
|
+
await this.okfTrustWrites.writeOkfTrust(entryId, entityId, [{ by, at: (/* @__PURE__ */ new Date()).toISOString() }], tx);
|
|
8833
|
+
});
|
|
8834
|
+
}
|
|
8601
8835
|
/** Set a fact's stale_after (epoch ms) or clear it. Does NOT touch `updated_at`. */
|
|
8602
8836
|
async setStaleAfter(entryId, entityId, date) {
|
|
8603
8837
|
return this.okfTrustWrites.setStaleAfter(entryId, entityId, date);
|
|
@@ -9549,6 +9783,7 @@ exports.ONTOLOGY_BACKFILL_SYSTEM_PROMPT = ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
|
|
|
9549
9783
|
exports.PromptService = PromptService;
|
|
9550
9784
|
exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
9551
9785
|
exports.WikiBusyError = WikiBusyError;
|
|
9786
|
+
exports.WikiDraftNotFound = WikiDraftNotFound;
|
|
9552
9787
|
exports.WikiDuplicateHashError = WikiDuplicateHashError;
|
|
9553
9788
|
exports.WikiGraphNodeOwnershipConflict = WikiGraphNodeOwnershipConflict;
|
|
9554
9789
|
exports.WikiIngestEmptyError = WikiIngestEmptyError;
|