@remnic/core 10.0.67 → 10.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +1 -1
- package/dist/{chunk-LFYEQK45.js → chunk-DDBB55L2.js} +417 -284
- package/dist/chunk-DDBB55L2.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/orchestration/buffer-drain-runner.js +1 -1
- package/dist/orchestration/extraction-persist.d.ts.map +1 -1
- package/dist/orchestration/promotion-dedup-window.d.ts +118 -0
- package/dist/orchestration/promotion-dedup-window.d.ts.map +1 -0
- package/dist/orchestrator.js +1 -1
- package/package.json +2 -2
- package/src/extraction-parallel-dedup.test.ts +185 -0
- package/src/extraction-shared-promotion-parallel-dedup.test.ts +212 -0
- package/src/orchestration/extraction-persist.ts +58 -61
- package/src/orchestration/promotion-dedup-window.ts +319 -0
- package/src/orchestration/provider-breaker.test.ts +22 -0
- package/dist/chunk-LFYEQK45.js.map +0 -1
|
@@ -5042,221 +5042,6 @@ async function persistUnderFactDedupWindow(storageDir, recheck, write) {
|
|
|
5042
5042
|
);
|
|
5043
5043
|
}
|
|
5044
5044
|
|
|
5045
|
-
// src/orchestration/extraction-injection-gate.ts
|
|
5046
|
-
function serializeInjectionScreenCandidate(candidate) {
|
|
5047
|
-
const body = candidate.category === "procedure" ? buildProcedurePersistBody(candidate.content, candidate.procedureSteps) : candidate.content;
|
|
5048
|
-
const attrs = Object.fromEntries(
|
|
5049
|
-
Object.entries(candidate.structuredAttributes ?? {}).flatMap(([key, value]) => {
|
|
5050
|
-
if (typeof value === "string") return [[key, value]];
|
|
5051
|
-
if (typeof value === "number" || typeof value === "boolean") return [[key, String(value)]];
|
|
5052
|
-
return [];
|
|
5053
|
-
})
|
|
5054
|
-
);
|
|
5055
|
-
return Object.keys(attrs).length > 0 ? `${body}
|
|
5056
|
-
[Attributes: ${normalizeAttributePairs(attrs)}]` : body;
|
|
5057
|
-
}
|
|
5058
|
-
function evaluateInjectionScreen(candidate, enabled, profile = "default") {
|
|
5059
|
-
if (!enabled) return { tags: [] };
|
|
5060
|
-
const content = typeof candidate === "string" ? candidate : serializeInjectionScreenCandidate(candidate);
|
|
5061
|
-
const result = screenCandidateFact(content, profile);
|
|
5062
|
-
return {
|
|
5063
|
-
status: result.quarantine === true ? "pending_review" : void 0,
|
|
5064
|
-
tags: result.quarantine === true ? result.findings.map((finding) => `injection-screen:${finding.rule}`) : []
|
|
5065
|
-
};
|
|
5066
|
-
}
|
|
5067
|
-
function withholdScreenedStrings(values, enabled, profile = "default") {
|
|
5068
|
-
const kept = [];
|
|
5069
|
-
const withheldRules = [];
|
|
5070
|
-
for (const value of values) {
|
|
5071
|
-
if (typeof value !== "string") continue;
|
|
5072
|
-
if (!enabled) {
|
|
5073
|
-
kept.push(value);
|
|
5074
|
-
continue;
|
|
5075
|
-
}
|
|
5076
|
-
const screened = screenCandidateFact(value, profile);
|
|
5077
|
-
if (screened.quarantine) {
|
|
5078
|
-
withheldRules.push(...screened.findings.map((finding) => `injection-screen:${finding.rule}`));
|
|
5079
|
-
} else {
|
|
5080
|
-
kept.push(value);
|
|
5081
|
-
}
|
|
5082
|
-
}
|
|
5083
|
-
return { kept, withheldRules };
|
|
5084
|
-
}
|
|
5085
|
-
function screenEntityForIndex(entity, enabled, profile = "default") {
|
|
5086
|
-
const record = entity;
|
|
5087
|
-
const name = record?.name;
|
|
5088
|
-
const type = record?.type;
|
|
5089
|
-
if (typeof name !== "string" || !name.trim() || typeof type !== "string" || !type.trim()) return null;
|
|
5090
|
-
const nameScreen = withholdScreenedStrings([name], enabled, profile);
|
|
5091
|
-
const factScreen = withholdScreenedStrings(Array.isArray(record?.facts) ? record.facts : [], enabled, profile);
|
|
5092
|
-
const rawSections = Array.isArray(record?.structuredSections) ? record.structuredSections : void 0;
|
|
5093
|
-
const sectionScreens = enabled && rawSections ? rawSections.map((section) => ({ section, screen: withholdScreenedStrings(section?.facts ?? [], true, profile) })) : void 0;
|
|
5094
|
-
return {
|
|
5095
|
-
name,
|
|
5096
|
-
type,
|
|
5097
|
-
source: typeof record?.source === "string" ? record.source : "extraction",
|
|
5098
|
-
facts: factScreen.kept,
|
|
5099
|
-
...sectionScreens ? {
|
|
5100
|
-
structuredSections: sectionScreens.map(({ section, screen }) => {
|
|
5101
|
-
let nextFactIndex = 0;
|
|
5102
|
-
const factOrigins = screen.kept.map((fact) => {
|
|
5103
|
-
const factIndex = section.facts.indexOf(fact, nextFactIndex);
|
|
5104
|
-
nextFactIndex = factIndex + 1;
|
|
5105
|
-
const origin = section.factOrigins?.[factIndex];
|
|
5106
|
-
return origin === void 0 ? void 0 : parseOriginClass(origin);
|
|
5107
|
-
});
|
|
5108
|
-
return {
|
|
5109
|
-
...section,
|
|
5110
|
-
facts: screen.kept,
|
|
5111
|
-
...factOrigins.some((origin) => origin !== void 0) ? { factOrigins } : {}
|
|
5112
|
-
};
|
|
5113
|
-
})
|
|
5114
|
-
} : rawSections ? { structuredSections: rawSections } : {},
|
|
5115
|
-
withheldRules: [
|
|
5116
|
-
...nameScreen.withheldRules,
|
|
5117
|
-
...factScreen.withheldRules,
|
|
5118
|
-
...(sectionScreens ?? []).flatMap(({ screen }) => screen.withheldRules)
|
|
5119
|
-
],
|
|
5120
|
-
withheld: nameScreen.withheldRules.length > 0
|
|
5121
|
-
};
|
|
5122
|
-
}
|
|
5123
|
-
function screenPersistStrings(values, enabled, profile = "default") {
|
|
5124
|
-
const screen = withholdScreenedStrings(values, enabled, profile);
|
|
5125
|
-
return {
|
|
5126
|
-
kept: screen.kept,
|
|
5127
|
-
...screen.withheldRules.length > 0 ? { warning: `injection screen withheld ${values.length - screen.kept.length} candidate(s) [${screen.withheldRules.join(", ")}]` } : {}
|
|
5128
|
-
};
|
|
5129
|
-
}
|
|
5130
|
-
|
|
5131
|
-
// src/orchestration/extraction-envelope.ts
|
|
5132
|
-
function composeSalvagedExtractionEnvelope(input, ctx) {
|
|
5133
|
-
const envelope = composeMemoryEnvelope(input, ctx, { salvage: true });
|
|
5134
|
-
if (envelope.salvageNotes.length > 0) {
|
|
5135
|
-
log.warn(`extraction write salvaged invalid fields: ${envelope.salvageNotes.join("; ")}`);
|
|
5136
|
-
}
|
|
5137
|
-
return envelope;
|
|
5138
|
-
}
|
|
5139
|
-
function withReservedMarkerTag(sourceTags, marker) {
|
|
5140
|
-
const usable = sourceTags.filter(
|
|
5141
|
-
(tag) => typeof tag !== "string" || tag.trim().length <= TAG_LIMITS.maxTagLength
|
|
5142
|
-
);
|
|
5143
|
-
const normalized = normalizeTags(usable) ?? [];
|
|
5144
|
-
const budget = TAG_LIMITS.maxTags - 1;
|
|
5145
|
-
if (normalized.length > budget) {
|
|
5146
|
-
log.warn(
|
|
5147
|
-
`extraction tags exceed ${budget}+marker cap; keeping the first ${budget} of ${normalized.length} and the ${JSON.stringify(marker)} marker`
|
|
5148
|
-
);
|
|
5149
|
-
}
|
|
5150
|
-
const capped = normalized.length > budget ? normalized.slice(0, budget) : normalized;
|
|
5151
|
-
return [...capped, marker];
|
|
5152
|
-
}
|
|
5153
|
-
function probeSalvageSurvivingFields(input) {
|
|
5154
|
-
try {
|
|
5155
|
-
const probe = composeSalvagedExtractionEnvelope(
|
|
5156
|
-
{
|
|
5157
|
-
content: input.content,
|
|
5158
|
-
category: input.category,
|
|
5159
|
-
structuredAttributes: input.structuredAttributes,
|
|
5160
|
-
entityRef: input.entityRef
|
|
5161
|
-
},
|
|
5162
|
-
{ source: "bitemporal-backfill-probe" }
|
|
5163
|
-
);
|
|
5164
|
-
return {
|
|
5165
|
-
entityRef: probe.entityRef,
|
|
5166
|
-
structuredAttributes: probe.rawStructuredAttributes ? { ...probe.rawStructuredAttributes } : void 0
|
|
5167
|
-
};
|
|
5168
|
-
} catch {
|
|
5169
|
-
return null;
|
|
5170
|
-
}
|
|
5171
|
-
}
|
|
5172
|
-
|
|
5173
|
-
// src/dedup/semantic.ts
|
|
5174
|
-
var DEFAULT_SEMANTIC_THRESHOLD = 0.92;
|
|
5175
|
-
var DEFAULT_SEMANTIC_CANDIDATES = 5;
|
|
5176
|
-
function normalizeSemanticThreshold(value) {
|
|
5177
|
-
return Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : DEFAULT_SEMANTIC_THRESHOLD;
|
|
5178
|
-
}
|
|
5179
|
-
function normalizeSemanticCandidates(value) {
|
|
5180
|
-
if (!Number.isFinite(value) || value < 0) {
|
|
5181
|
-
return DEFAULT_SEMANTIC_CANDIDATES;
|
|
5182
|
-
}
|
|
5183
|
-
const normalized = Math.floor(value);
|
|
5184
|
-
return value > 0 && normalized === 0 ? 1 : normalized;
|
|
5185
|
-
}
|
|
5186
|
-
async function decideSemanticDedup(content, lookup, options) {
|
|
5187
|
-
if (!options.enabled) {
|
|
5188
|
-
return { action: "keep", reason: "disabled" };
|
|
5189
|
-
}
|
|
5190
|
-
const threshold = normalizeSemanticThreshold(options.threshold);
|
|
5191
|
-
const candidates = normalizeSemanticCandidates(options.candidates);
|
|
5192
|
-
if (candidates === 0) {
|
|
5193
|
-
return { action: "keep", reason: "disabled" };
|
|
5194
|
-
}
|
|
5195
|
-
const trimmed = typeof content === "string" ? content.trim() : "";
|
|
5196
|
-
if (!trimmed) {
|
|
5197
|
-
return { action: "keep", reason: "no_near_duplicate" };
|
|
5198
|
-
}
|
|
5199
|
-
let hits = [];
|
|
5200
|
-
try {
|
|
5201
|
-
hits = await lookup(trimmed, candidates);
|
|
5202
|
-
} catch {
|
|
5203
|
-
return { action: "keep", reason: "backend_unavailable" };
|
|
5204
|
-
}
|
|
5205
|
-
if (!Array.isArray(hits) || hits.length === 0) {
|
|
5206
|
-
return { action: "keep", reason: "no_candidates" };
|
|
5207
|
-
}
|
|
5208
|
-
const candidateConnector = normalizeConnectorScope(options.sourceConnector);
|
|
5209
|
-
let top;
|
|
5210
|
-
let sameConnectorTop;
|
|
5211
|
-
for (const hit of hits) {
|
|
5212
|
-
if (!hit || typeof hit.id !== "string" || hit.id.trim().length === 0 || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
|
|
5213
|
-
continue;
|
|
5214
|
-
}
|
|
5215
|
-
if (!top || hit.score > top.score) {
|
|
5216
|
-
top = hit;
|
|
5217
|
-
}
|
|
5218
|
-
if (candidateConnector !== void 0 && connectorMatchesScope(hit.sourceConnector, candidateConnector) && (!sameConnectorTop || hit.score > sameConnectorTop.score)) {
|
|
5219
|
-
sameConnectorTop = hit;
|
|
5220
|
-
}
|
|
5221
|
-
}
|
|
5222
|
-
if (!top) {
|
|
5223
|
-
return { action: "keep", reason: "no_near_duplicate" };
|
|
5224
|
-
}
|
|
5225
|
-
if (top.score >= threshold) {
|
|
5226
|
-
if (candidateConnector !== void 0) {
|
|
5227
|
-
const match = sameConnectorTop !== void 0 && sameConnectorTop.score >= threshold ? sameConnectorTop : void 0;
|
|
5228
|
-
if (match) {
|
|
5229
|
-
return {
|
|
5230
|
-
action: "skip",
|
|
5231
|
-
reason: "near_duplicate",
|
|
5232
|
-
topScore: match.score,
|
|
5233
|
-
topId: match.id,
|
|
5234
|
-
topPath: match.path
|
|
5235
|
-
};
|
|
5236
|
-
}
|
|
5237
|
-
return {
|
|
5238
|
-
action: "keep",
|
|
5239
|
-
reason: "connector_mismatch",
|
|
5240
|
-
topScore: top.score,
|
|
5241
|
-
topId: top.id
|
|
5242
|
-
};
|
|
5243
|
-
}
|
|
5244
|
-
return {
|
|
5245
|
-
action: "skip",
|
|
5246
|
-
reason: "near_duplicate",
|
|
5247
|
-
topScore: top.score,
|
|
5248
|
-
topId: top.id,
|
|
5249
|
-
topPath: top.path
|
|
5250
|
-
};
|
|
5251
|
-
}
|
|
5252
|
-
return {
|
|
5253
|
-
action: "keep",
|
|
5254
|
-
reason: "no_near_duplicate",
|
|
5255
|
-
topScore: top.score,
|
|
5256
|
-
topId: top.id
|
|
5257
|
-
};
|
|
5258
|
-
}
|
|
5259
|
-
|
|
5260
5045
|
// src/orchestration/semantic-merge-promotion-payload.ts
|
|
5261
5046
|
import { createHash as createHash3 } from "crypto";
|
|
5262
5047
|
import { readFile as readFile2 } from "fs/promises";
|
|
@@ -5717,6 +5502,356 @@ async function retireStaleMergedTargetPromotionCopies(args) {
|
|
|
5717
5502
|
return retired;
|
|
5718
5503
|
}
|
|
5719
5504
|
|
|
5505
|
+
// src/orchestration/promotion-dedup-window.ts
|
|
5506
|
+
async function findPromotionRaceDuplicate(args) {
|
|
5507
|
+
try {
|
|
5508
|
+
const allMems = await readActiveMemoriesBothTiers(args.storage);
|
|
5509
|
+
const nc = args.connector?.trim() || void 0;
|
|
5510
|
+
const incomingNormalized = ContentHashIndex.normalizeContent(args.dedupContent);
|
|
5511
|
+
return allMems.find((m) => {
|
|
5512
|
+
if (m.frontmatter.category !== args.category) return false;
|
|
5513
|
+
if ((m.frontmatter.status ?? "active") !== "active") return false;
|
|
5514
|
+
if (normalizeStoredHashSource(m.content ?? "", args.citationEnabled, args.citationTemplate) !== incomingNormalized)
|
|
5515
|
+
return false;
|
|
5516
|
+
if (args.entityKeyNorm !== void 0) {
|
|
5517
|
+
if (!m.frontmatter.entityRef) return false;
|
|
5518
|
+
if (normalizeSupersessionKey(m.frontmatter.entityRef) !== args.entityKeyNorm) return false;
|
|
5519
|
+
}
|
|
5520
|
+
return (m.frontmatter.sourceConnector?.trim() || void 0) === nc;
|
|
5521
|
+
});
|
|
5522
|
+
} catch (err) {
|
|
5523
|
+
log.warn(
|
|
5524
|
+
`promotion race recheck scan failed for ${args.storage.dir}; writing fail-open: ${err instanceof Error ? err.message : String(err)}`
|
|
5525
|
+
);
|
|
5526
|
+
return void 0;
|
|
5527
|
+
}
|
|
5528
|
+
}
|
|
5529
|
+
async function confirmPromotionRaceDuplicate(args) {
|
|
5530
|
+
try {
|
|
5531
|
+
if (await args.storage.isFactContentHashAuthoritative()) {
|
|
5532
|
+
if (!await args.storage.hasFactContentHash(args.dedupContent)) return false;
|
|
5533
|
+
}
|
|
5534
|
+
} catch (probeErr) {
|
|
5535
|
+
log.warn(
|
|
5536
|
+
`promotion race recheck authoritative probe failed for ${args.storage.dir}; confirming against corpus: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`
|
|
5537
|
+
);
|
|
5538
|
+
}
|
|
5539
|
+
return await findPromotionRaceDuplicate(args) !== void 0;
|
|
5540
|
+
}
|
|
5541
|
+
function buildPromotionWriteOptions(args) {
|
|
5542
|
+
const { promotion } = args;
|
|
5543
|
+
const writeOptions = {
|
|
5544
|
+
importance: promotion.importance,
|
|
5545
|
+
lineage: [promotion.sourceMemoryId],
|
|
5546
|
+
sourceMemoryId: promotion.sourceMemoryId,
|
|
5547
|
+
intentGoal: promotion.intentGoal,
|
|
5548
|
+
intentActionType: promotion.intentActionType,
|
|
5549
|
+
intentEntityTypes: promotion.intentEntityTypes,
|
|
5550
|
+
memoryKind: promotion.memoryKind,
|
|
5551
|
+
contentHashSource: promotion.category === "fact" ? args.dedupContent : args.rawContent
|
|
5552
|
+
};
|
|
5553
|
+
if (promotion.invalidAt) writeOptions.invalidAt = promotion.invalidAt;
|
|
5554
|
+
if (promotion.observedAt) writeOptions.observedAt = promotion.observedAt;
|
|
5555
|
+
if (promotion.eventTimeSource) writeOptions.eventTimeSource = promotion.eventTimeSource;
|
|
5556
|
+
if (promotion.sources && promotion.sources.length > 0) writeOptions.sources = promotion.sources;
|
|
5557
|
+
if (promotion.provenance) writeOptions.provenance = promotion.provenance;
|
|
5558
|
+
if (args.toolScoped) writeOptions.toolScoped = true;
|
|
5559
|
+
return writeOptions;
|
|
5560
|
+
}
|
|
5561
|
+
async function writePromotionUnderFactDedupWindow(args) {
|
|
5562
|
+
const { destination, envelope, promotion } = args;
|
|
5563
|
+
if (promotion.category !== "fact") {
|
|
5564
|
+
return {
|
|
5565
|
+
suppressed: false,
|
|
5566
|
+
result: await destination.writeSealedMemory(envelope, buildPromotionWriteOptions(args))
|
|
5567
|
+
};
|
|
5568
|
+
}
|
|
5569
|
+
const entityKeyNorm = args.sameEntityGuardEnabled && envelope.entityRef && args.survivingAttrs && Object.keys(args.survivingAttrs).length > 0 ? normalizeSupersessionKey(envelope.entityRef) : void 0;
|
|
5570
|
+
const outcome = await persistUnderFactDedupWindow(
|
|
5571
|
+
destination.dir,
|
|
5572
|
+
() => promotion.category === "fact" ? confirmPromotionRaceDuplicate({
|
|
5573
|
+
storage: destination,
|
|
5574
|
+
dedupContent: args.dedupContent,
|
|
5575
|
+
category: promotion.category,
|
|
5576
|
+
connector: args.connector,
|
|
5577
|
+
citationEnabled: args.citationEnabled,
|
|
5578
|
+
citationTemplate: args.citationTemplate,
|
|
5579
|
+
entityKeyNorm
|
|
5580
|
+
}) : Promise.resolve(false),
|
|
5581
|
+
() => destination.writeSealedMemory(envelope, buildPromotionWriteOptions(args))
|
|
5582
|
+
);
|
|
5583
|
+
if (outcome.suppressed) {
|
|
5584
|
+
log.info(`dedup: raced promotion duplicate suppressed in fact-dedup window for ${destination.dir}`);
|
|
5585
|
+
if (promotion.invalidAt || promotion.validAt) {
|
|
5586
|
+
await args.backfillTemporalBoundsOnDedupHit(
|
|
5587
|
+
destination,
|
|
5588
|
+
args.dedupContent,
|
|
5589
|
+
{
|
|
5590
|
+
invalidAt: promotion.invalidAt,
|
|
5591
|
+
// #1707 thread 2 — carry the corrected start bound.
|
|
5592
|
+
validFrom: promotion.validAt,
|
|
5593
|
+
observedAt: promotion.observedAt,
|
|
5594
|
+
eventTimeSource: promotion.eventTimeSource
|
|
5595
|
+
},
|
|
5596
|
+
envelope.entityRef,
|
|
5597
|
+
args.connector
|
|
5598
|
+
);
|
|
5599
|
+
}
|
|
5600
|
+
if (args.supersession?.supersessionEnabled && promotion.category === "fact" && envelope.entityRef && args.survivingAttrs && Object.keys(args.survivingAttrs).length > 0) {
|
|
5601
|
+
try {
|
|
5602
|
+
const winning = await findPromotionRaceDuplicate({
|
|
5603
|
+
storage: destination,
|
|
5604
|
+
dedupContent: args.dedupContent,
|
|
5605
|
+
category: promotion.category,
|
|
5606
|
+
connector: args.connector,
|
|
5607
|
+
citationEnabled: args.citationEnabled,
|
|
5608
|
+
citationTemplate: args.citationTemplate,
|
|
5609
|
+
// PRRT_kwDORJXyws6z2o11: the winner ANCHORS the pass, so the lookup
|
|
5610
|
+
// is always entity-guarded — without it (profile path) it could
|
|
5611
|
+
// match another entity's equal-content fact and combine that fact's
|
|
5612
|
+
// id with the incoming entity's attributes (PR #402 Uybg hazard).
|
|
5613
|
+
entityKeyNorm: normalizeSupersessionKey(envelope.entityRef)
|
|
5614
|
+
});
|
|
5615
|
+
if (winning) {
|
|
5616
|
+
await applyTemporalSupersession({
|
|
5617
|
+
storage: destination,
|
|
5618
|
+
newMemoryId: winning.frontmatter.id,
|
|
5619
|
+
entityRef: envelope.entityRef,
|
|
5620
|
+
structuredAttributes: { ...args.survivingAttrs },
|
|
5621
|
+
// Ordering anchor mirrors the gate's supersessionOrderingAt: an
|
|
5622
|
+
// incoming event date orders the pass, wall clock otherwise.
|
|
5623
|
+
createdAt: promotion.validAt && promotion.validAt.length > 0 ? promotion.validAt : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5624
|
+
useCallerTimestamp: true,
|
|
5625
|
+
enabled: !(promotion.eventTimeSource === "extracted" && !promotion.validAt),
|
|
5626
|
+
namespaceScope: args.supersession.namespaceScope,
|
|
5627
|
+
dependencyPropagationDelivery: args.supersession.dependencyPropagationDelivery
|
|
5628
|
+
});
|
|
5629
|
+
}
|
|
5630
|
+
} catch (supersessionErr) {
|
|
5631
|
+
log.warn(
|
|
5632
|
+
`promotion window supersession failed open for ${destination.dir}: ${supersessionErr instanceof Error ? supersessionErr.message : String(supersessionErr)}`
|
|
5633
|
+
);
|
|
5634
|
+
}
|
|
5635
|
+
}
|
|
5636
|
+
}
|
|
5637
|
+
return outcome;
|
|
5638
|
+
}
|
|
5639
|
+
|
|
5640
|
+
// src/orchestration/extraction-injection-gate.ts
|
|
5641
|
+
function serializeInjectionScreenCandidate(candidate) {
|
|
5642
|
+
const body = candidate.category === "procedure" ? buildProcedurePersistBody(candidate.content, candidate.procedureSteps) : candidate.content;
|
|
5643
|
+
const attrs = Object.fromEntries(
|
|
5644
|
+
Object.entries(candidate.structuredAttributes ?? {}).flatMap(([key, value]) => {
|
|
5645
|
+
if (typeof value === "string") return [[key, value]];
|
|
5646
|
+
if (typeof value === "number" || typeof value === "boolean") return [[key, String(value)]];
|
|
5647
|
+
return [];
|
|
5648
|
+
})
|
|
5649
|
+
);
|
|
5650
|
+
return Object.keys(attrs).length > 0 ? `${body}
|
|
5651
|
+
[Attributes: ${normalizeAttributePairs(attrs)}]` : body;
|
|
5652
|
+
}
|
|
5653
|
+
function evaluateInjectionScreen(candidate, enabled, profile = "default") {
|
|
5654
|
+
if (!enabled) return { tags: [] };
|
|
5655
|
+
const content = typeof candidate === "string" ? candidate : serializeInjectionScreenCandidate(candidate);
|
|
5656
|
+
const result = screenCandidateFact(content, profile);
|
|
5657
|
+
return {
|
|
5658
|
+
status: result.quarantine === true ? "pending_review" : void 0,
|
|
5659
|
+
tags: result.quarantine === true ? result.findings.map((finding) => `injection-screen:${finding.rule}`) : []
|
|
5660
|
+
};
|
|
5661
|
+
}
|
|
5662
|
+
function withholdScreenedStrings(values, enabled, profile = "default") {
|
|
5663
|
+
const kept = [];
|
|
5664
|
+
const withheldRules = [];
|
|
5665
|
+
for (const value of values) {
|
|
5666
|
+
if (typeof value !== "string") continue;
|
|
5667
|
+
if (!enabled) {
|
|
5668
|
+
kept.push(value);
|
|
5669
|
+
continue;
|
|
5670
|
+
}
|
|
5671
|
+
const screened = screenCandidateFact(value, profile);
|
|
5672
|
+
if (screened.quarantine) {
|
|
5673
|
+
withheldRules.push(...screened.findings.map((finding) => `injection-screen:${finding.rule}`));
|
|
5674
|
+
} else {
|
|
5675
|
+
kept.push(value);
|
|
5676
|
+
}
|
|
5677
|
+
}
|
|
5678
|
+
return { kept, withheldRules };
|
|
5679
|
+
}
|
|
5680
|
+
function screenEntityForIndex(entity, enabled, profile = "default") {
|
|
5681
|
+
const record = entity;
|
|
5682
|
+
const name = record?.name;
|
|
5683
|
+
const type = record?.type;
|
|
5684
|
+
if (typeof name !== "string" || !name.trim() || typeof type !== "string" || !type.trim()) return null;
|
|
5685
|
+
const nameScreen = withholdScreenedStrings([name], enabled, profile);
|
|
5686
|
+
const factScreen = withholdScreenedStrings(Array.isArray(record?.facts) ? record.facts : [], enabled, profile);
|
|
5687
|
+
const rawSections = Array.isArray(record?.structuredSections) ? record.structuredSections : void 0;
|
|
5688
|
+
const sectionScreens = enabled && rawSections ? rawSections.map((section) => ({ section, screen: withholdScreenedStrings(section?.facts ?? [], true, profile) })) : void 0;
|
|
5689
|
+
return {
|
|
5690
|
+
name,
|
|
5691
|
+
type,
|
|
5692
|
+
source: typeof record?.source === "string" ? record.source : "extraction",
|
|
5693
|
+
facts: factScreen.kept,
|
|
5694
|
+
...sectionScreens ? {
|
|
5695
|
+
structuredSections: sectionScreens.map(({ section, screen }) => {
|
|
5696
|
+
let nextFactIndex = 0;
|
|
5697
|
+
const factOrigins = screen.kept.map((fact) => {
|
|
5698
|
+
const factIndex = section.facts.indexOf(fact, nextFactIndex);
|
|
5699
|
+
nextFactIndex = factIndex + 1;
|
|
5700
|
+
const origin = section.factOrigins?.[factIndex];
|
|
5701
|
+
return origin === void 0 ? void 0 : parseOriginClass(origin);
|
|
5702
|
+
});
|
|
5703
|
+
return {
|
|
5704
|
+
...section,
|
|
5705
|
+
facts: screen.kept,
|
|
5706
|
+
...factOrigins.some((origin) => origin !== void 0) ? { factOrigins } : {}
|
|
5707
|
+
};
|
|
5708
|
+
})
|
|
5709
|
+
} : rawSections ? { structuredSections: rawSections } : {},
|
|
5710
|
+
withheldRules: [
|
|
5711
|
+
...nameScreen.withheldRules,
|
|
5712
|
+
...factScreen.withheldRules,
|
|
5713
|
+
...(sectionScreens ?? []).flatMap(({ screen }) => screen.withheldRules)
|
|
5714
|
+
],
|
|
5715
|
+
withheld: nameScreen.withheldRules.length > 0
|
|
5716
|
+
};
|
|
5717
|
+
}
|
|
5718
|
+
function screenPersistStrings(values, enabled, profile = "default") {
|
|
5719
|
+
const screen = withholdScreenedStrings(values, enabled, profile);
|
|
5720
|
+
return {
|
|
5721
|
+
kept: screen.kept,
|
|
5722
|
+
...screen.withheldRules.length > 0 ? { warning: `injection screen withheld ${values.length - screen.kept.length} candidate(s) [${screen.withheldRules.join(", ")}]` } : {}
|
|
5723
|
+
};
|
|
5724
|
+
}
|
|
5725
|
+
|
|
5726
|
+
// src/orchestration/extraction-envelope.ts
|
|
5727
|
+
function composeSalvagedExtractionEnvelope(input, ctx) {
|
|
5728
|
+
const envelope = composeMemoryEnvelope(input, ctx, { salvage: true });
|
|
5729
|
+
if (envelope.salvageNotes.length > 0) {
|
|
5730
|
+
log.warn(`extraction write salvaged invalid fields: ${envelope.salvageNotes.join("; ")}`);
|
|
5731
|
+
}
|
|
5732
|
+
return envelope;
|
|
5733
|
+
}
|
|
5734
|
+
function withReservedMarkerTag(sourceTags, marker) {
|
|
5735
|
+
const usable = sourceTags.filter(
|
|
5736
|
+
(tag) => typeof tag !== "string" || tag.trim().length <= TAG_LIMITS.maxTagLength
|
|
5737
|
+
);
|
|
5738
|
+
const normalized = normalizeTags(usable) ?? [];
|
|
5739
|
+
const budget = TAG_LIMITS.maxTags - 1;
|
|
5740
|
+
if (normalized.length > budget) {
|
|
5741
|
+
log.warn(
|
|
5742
|
+
`extraction tags exceed ${budget}+marker cap; keeping the first ${budget} of ${normalized.length} and the ${JSON.stringify(marker)} marker`
|
|
5743
|
+
);
|
|
5744
|
+
}
|
|
5745
|
+
const capped = normalized.length > budget ? normalized.slice(0, budget) : normalized;
|
|
5746
|
+
return [...capped, marker];
|
|
5747
|
+
}
|
|
5748
|
+
function probeSalvageSurvivingFields(input) {
|
|
5749
|
+
try {
|
|
5750
|
+
const probe = composeSalvagedExtractionEnvelope(
|
|
5751
|
+
{
|
|
5752
|
+
content: input.content,
|
|
5753
|
+
category: input.category,
|
|
5754
|
+
structuredAttributes: input.structuredAttributes,
|
|
5755
|
+
entityRef: input.entityRef
|
|
5756
|
+
},
|
|
5757
|
+
{ source: "bitemporal-backfill-probe" }
|
|
5758
|
+
);
|
|
5759
|
+
return {
|
|
5760
|
+
entityRef: probe.entityRef,
|
|
5761
|
+
structuredAttributes: probe.rawStructuredAttributes ? { ...probe.rawStructuredAttributes } : void 0
|
|
5762
|
+
};
|
|
5763
|
+
} catch {
|
|
5764
|
+
return null;
|
|
5765
|
+
}
|
|
5766
|
+
}
|
|
5767
|
+
|
|
5768
|
+
// src/dedup/semantic.ts
|
|
5769
|
+
var DEFAULT_SEMANTIC_THRESHOLD = 0.92;
|
|
5770
|
+
var DEFAULT_SEMANTIC_CANDIDATES = 5;
|
|
5771
|
+
function normalizeSemanticThreshold(value) {
|
|
5772
|
+
return Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : DEFAULT_SEMANTIC_THRESHOLD;
|
|
5773
|
+
}
|
|
5774
|
+
function normalizeSemanticCandidates(value) {
|
|
5775
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
5776
|
+
return DEFAULT_SEMANTIC_CANDIDATES;
|
|
5777
|
+
}
|
|
5778
|
+
const normalized = Math.floor(value);
|
|
5779
|
+
return value > 0 && normalized === 0 ? 1 : normalized;
|
|
5780
|
+
}
|
|
5781
|
+
async function decideSemanticDedup(content, lookup, options) {
|
|
5782
|
+
if (!options.enabled) {
|
|
5783
|
+
return { action: "keep", reason: "disabled" };
|
|
5784
|
+
}
|
|
5785
|
+
const threshold = normalizeSemanticThreshold(options.threshold);
|
|
5786
|
+
const candidates = normalizeSemanticCandidates(options.candidates);
|
|
5787
|
+
if (candidates === 0) {
|
|
5788
|
+
return { action: "keep", reason: "disabled" };
|
|
5789
|
+
}
|
|
5790
|
+
const trimmed = typeof content === "string" ? content.trim() : "";
|
|
5791
|
+
if (!trimmed) {
|
|
5792
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
5793
|
+
}
|
|
5794
|
+
let hits = [];
|
|
5795
|
+
try {
|
|
5796
|
+
hits = await lookup(trimmed, candidates);
|
|
5797
|
+
} catch {
|
|
5798
|
+
return { action: "keep", reason: "backend_unavailable" };
|
|
5799
|
+
}
|
|
5800
|
+
if (!Array.isArray(hits) || hits.length === 0) {
|
|
5801
|
+
return { action: "keep", reason: "no_candidates" };
|
|
5802
|
+
}
|
|
5803
|
+
const candidateConnector = normalizeConnectorScope(options.sourceConnector);
|
|
5804
|
+
let top;
|
|
5805
|
+
let sameConnectorTop;
|
|
5806
|
+
for (const hit of hits) {
|
|
5807
|
+
if (!hit || typeof hit.id !== "string" || hit.id.trim().length === 0 || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
|
|
5808
|
+
continue;
|
|
5809
|
+
}
|
|
5810
|
+
if (!top || hit.score > top.score) {
|
|
5811
|
+
top = hit;
|
|
5812
|
+
}
|
|
5813
|
+
if (candidateConnector !== void 0 && connectorMatchesScope(hit.sourceConnector, candidateConnector) && (!sameConnectorTop || hit.score > sameConnectorTop.score)) {
|
|
5814
|
+
sameConnectorTop = hit;
|
|
5815
|
+
}
|
|
5816
|
+
}
|
|
5817
|
+
if (!top) {
|
|
5818
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
5819
|
+
}
|
|
5820
|
+
if (top.score >= threshold) {
|
|
5821
|
+
if (candidateConnector !== void 0) {
|
|
5822
|
+
const match = sameConnectorTop !== void 0 && sameConnectorTop.score >= threshold ? sameConnectorTop : void 0;
|
|
5823
|
+
if (match) {
|
|
5824
|
+
return {
|
|
5825
|
+
action: "skip",
|
|
5826
|
+
reason: "near_duplicate",
|
|
5827
|
+
topScore: match.score,
|
|
5828
|
+
topId: match.id,
|
|
5829
|
+
topPath: match.path
|
|
5830
|
+
};
|
|
5831
|
+
}
|
|
5832
|
+
return {
|
|
5833
|
+
action: "keep",
|
|
5834
|
+
reason: "connector_mismatch",
|
|
5835
|
+
topScore: top.score,
|
|
5836
|
+
topId: top.id
|
|
5837
|
+
};
|
|
5838
|
+
}
|
|
5839
|
+
return {
|
|
5840
|
+
action: "skip",
|
|
5841
|
+
reason: "near_duplicate",
|
|
5842
|
+
topScore: top.score,
|
|
5843
|
+
topId: top.id,
|
|
5844
|
+
topPath: top.path
|
|
5845
|
+
};
|
|
5846
|
+
}
|
|
5847
|
+
return {
|
|
5848
|
+
action: "keep",
|
|
5849
|
+
reason: "no_near_duplicate",
|
|
5850
|
+
topScore: top.score,
|
|
5851
|
+
topId: top.id
|
|
5852
|
+
};
|
|
5853
|
+
}
|
|
5854
|
+
|
|
5720
5855
|
// src/orchestration/semantic-merge-persist.ts
|
|
5721
5856
|
import path6 from "path";
|
|
5722
5857
|
|
|
@@ -6736,23 +6871,23 @@ var ExtractionPersistCoordinator = class {
|
|
|
6736
6871
|
continue;
|
|
6737
6872
|
}
|
|
6738
6873
|
}
|
|
6739
|
-
const
|
|
6740
|
-
|
|
6741
|
-
|
|
6742
|
-
|
|
6743
|
-
|
|
6744
|
-
|
|
6745
|
-
|
|
6746
|
-
|
|
6747
|
-
|
|
6748
|
-
|
|
6749
|
-
|
|
6750
|
-
|
|
6751
|
-
|
|
6752
|
-
|
|
6753
|
-
...options.provenance ? { provenance: options.provenance } : {},
|
|
6754
|
-
...options.toolScoped ? { toolScoped: true } : {}
|
|
6874
|
+
const promotionWrite = await writePromotionUnderFactDedupWindow({
|
|
6875
|
+
destination: targetStorage,
|
|
6876
|
+
envelope: targetPromotionEnvelope,
|
|
6877
|
+
promotion: options,
|
|
6878
|
+
dedupContent,
|
|
6879
|
+
rawContent,
|
|
6880
|
+
toolScoped: options.toolScoped === true,
|
|
6881
|
+
connector: sourceContext?.sourceConnector,
|
|
6882
|
+
citationEnabled,
|
|
6883
|
+
citationTemplate,
|
|
6884
|
+
sameEntityGuardEnabled: false,
|
|
6885
|
+
survivingAttrs: targetSurvivingAttrs,
|
|
6886
|
+
supersession: { supersessionEnabled: lifecycleCaps.temporalSupersession, dependencyPropagationDelivery: this.dependencyPropagationDelivery, namespaceScope: this.deps.storageDirNamespace(targetStorage.dir) },
|
|
6887
|
+
backfillTemporalBoundsOnDedupHit: this.deps.backfillTemporalBoundsOnDedupHit
|
|
6755
6888
|
});
|
|
6889
|
+
if (promotionWrite.suppressed) continue;
|
|
6890
|
+
const targetPromotion = promotionWrite.result;
|
|
6756
6891
|
const promotedId = targetPromotion.id;
|
|
6757
6892
|
if (!targetPromotion.tombstoneBlocked && lifecycleCaps.temporalSupersession && options.category === "fact" && targetPromotionEnvelope.entityRef && targetSurvivingAttrs && Object.keys(targetSurvivingAttrs).length > 0) {
|
|
6758
6893
|
try {
|
|
@@ -6956,24 +7091,23 @@ var ExtractionPersistCoordinator = class {
|
|
|
6956
7091
|
}
|
|
6957
7092
|
}
|
|
6958
7093
|
}
|
|
6959
|
-
const
|
|
6960
|
-
|
|
6961
|
-
|
|
6962
|
-
|
|
6963
|
-
|
|
6964
|
-
|
|
6965
|
-
|
|
6966
|
-
|
|
6967
|
-
|
|
6968
|
-
|
|
6969
|
-
|
|
6970
|
-
|
|
6971
|
-
|
|
6972
|
-
|
|
6973
|
-
...options.sources && options.sources.length > 0 ? { sources: options.sources } : {},
|
|
6974
|
-
...options.provenance ? { provenance: options.provenance } : {},
|
|
6975
|
-
...toolScoped ? { toolScoped: true } : {}
|
|
7094
|
+
const sharedWrite = await writePromotionUnderFactDedupWindow({
|
|
7095
|
+
destination: sharedStorage,
|
|
7096
|
+
envelope: sharedPromotionEnvelope,
|
|
7097
|
+
promotion: options,
|
|
7098
|
+
dedupContent,
|
|
7099
|
+
rawContent,
|
|
7100
|
+
toolScoped,
|
|
7101
|
+
connector: sourceContext?.sourceConnector,
|
|
7102
|
+
citationEnabled,
|
|
7103
|
+
citationTemplate,
|
|
7104
|
+
sameEntityGuardEnabled: lifecycleCaps.temporalSupersession,
|
|
7105
|
+
survivingAttrs: sharedSurvivingAttrs,
|
|
7106
|
+
supersession: { supersessionEnabled: lifecycleCaps.temporalSupersession, dependencyPropagationDelivery: this.dependencyPropagationDelivery, namespaceScope: this.deps.storageDirNamespace(sharedStorage.dir) },
|
|
7107
|
+
backfillTemporalBoundsOnDedupHit: this.deps.backfillTemporalBoundsOnDedupHit
|
|
6976
7108
|
});
|
|
7109
|
+
if (sharedWrite.suppressed) return promotedCopyId;
|
|
7110
|
+
const sharedPromotion = sharedWrite.result;
|
|
6977
7111
|
const promotedId = sharedPromotion.id;
|
|
6978
7112
|
if (!sharedPromotion.tombstoneBlocked && lifecycleCaps.temporalSupersession && sharedPromotionEnvelope.entityRef && sharedSurvivingAttrs && Object.keys(sharedSurvivingAttrs).length > 0) {
|
|
6979
7113
|
try {
|
|
@@ -8059,30 +8193,40 @@ var ExtractionPersistCoordinator = class {
|
|
|
8059
8193
|
const factWriteOutcome = await persistUnderFactDedupWindow(
|
|
8060
8194
|
targetStorage.dir,
|
|
8061
8195
|
raceRecheck(targetStorage, contentHashDedupKey),
|
|
8062
|
-
async () =>
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
|
|
8068
|
-
|
|
8069
|
-
|
|
8070
|
-
|
|
8071
|
-
|
|
8072
|
-
|
|
8073
|
-
|
|
8074
|
-
|
|
8075
|
-
|
|
8076
|
-
|
|
8077
|
-
|
|
8078
|
-
|
|
8079
|
-
|
|
8080
|
-
|
|
8081
|
-
|
|
8082
|
-
|
|
8083
|
-
|
|
8084
|
-
|
|
8085
|
-
|
|
8196
|
+
async () => {
|
|
8197
|
+
const factWrite2 = await targetStorage.writeSealedMemory(factWriteEnvelope, {
|
|
8198
|
+
importance,
|
|
8199
|
+
supersedes,
|
|
8200
|
+
links: links.length > 0 ? links : void 0,
|
|
8201
|
+
intentGoal: inferredIntent?.goal,
|
|
8202
|
+
intentActionType: inferredIntent?.actionType,
|
|
8203
|
+
intentEntityTypes: inferredIntent?.entityTypes,
|
|
8204
|
+
memoryKind,
|
|
8205
|
+
...biTemporal ? { observedAt: biTemporal.observedAt, eventTimeSource: biTemporal.eventTimeSource, ...biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {} } : {},
|
|
8206
|
+
contentHashSource: writeCategory === "fact" ? fact.content : void 0,
|
|
8207
|
+
// Faithfulness gate (issue #1576).
|
|
8208
|
+
...faithfulnessFm ? { faithfulness: faithfulnessFm } : {},
|
|
8209
|
+
...faithfulnessEnforceStatus ? { status: faithfulnessEnforceStatus } : {},
|
|
8210
|
+
// Claim-level provenance spans (issue #1575 PR 2). Carry verified
|
|
8211
|
+
// sources + the coarse strength tag from the extraction validator
|
|
8212
|
+
// through to frontmatter so they survive end-to-end.
|
|
8213
|
+
...fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {},
|
|
8214
|
+
...fact.provenance ? { provenance: fact.provenance } : {},
|
|
8215
|
+
// PR #2016: never defer inside the commit window (chunked-path
|
|
8216
|
+
// rule): a peer loading an authoritative index after the release
|
|
8217
|
+
// must see this hash, or its recheck misses and duplicates it.
|
|
8218
|
+
deferHashIndexSave: false,
|
|
8219
|
+
...factToolScoped ? { toolScoped: true } : {}
|
|
8220
|
+
});
|
|
8221
|
+
if (!factWrite2.tombstoneBlocked) {
|
|
8222
|
+
try {
|
|
8223
|
+
await this.deps.addContentHashDedup(targetStorage, contentHashDedupKey);
|
|
8224
|
+
} catch (err) {
|
|
8225
|
+
log.warn(`content-hash dedup registration failed for memory ${factWrite2.id}: ${err}`);
|
|
8226
|
+
}
|
|
8227
|
+
}
|
|
8228
|
+
return factWrite2;
|
|
8229
|
+
}
|
|
8086
8230
|
).catch(finalizeOnWindowFailure);
|
|
8087
8231
|
if (factWriteOutcome.suppressed) {
|
|
8088
8232
|
await backfillDuplicateTemporalBounds();
|
|
@@ -8239,17 +8383,6 @@ var ExtractionPersistCoordinator = class {
|
|
|
8239
8383
|
...factToolScoped ? { toolScoped: true } : {}
|
|
8240
8384
|
});
|
|
8241
8385
|
}
|
|
8242
|
-
try {
|
|
8243
|
-
const canonicalFactContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
8244
|
-
const hashRegisterKey = writeCategory === "procedure" ? buildProcedurePersistBody(fact.content, fact.procedureSteps) : canonicalFactContent;
|
|
8245
|
-
if (!tombstoneBlocked) {
|
|
8246
|
-
await this.deps.addContentHashDedup(targetStorage, hashRegisterKey);
|
|
8247
|
-
}
|
|
8248
|
-
} catch (err) {
|
|
8249
|
-
log.warn(
|
|
8250
|
-
`content-hash dedup registration failed for memory ${memoryId}: ${err}`
|
|
8251
|
-
);
|
|
8252
|
-
}
|
|
8253
8386
|
} catch (err) {
|
|
8254
8387
|
await finalizeCommittedWrites(true).catch(
|
|
8255
8388
|
(finalizeErr) => log.warn(`committed-write finalization after post-write failure failed: ${finalizeErr}`)
|
|
@@ -31777,4 +31910,4 @@ export {
|
|
|
31777
31910
|
qmdStartupCollectionChecksWithTimeout,
|
|
31778
31911
|
Orchestrator
|
|
31779
31912
|
};
|
|
31780
|
-
//# sourceMappingURL=chunk-
|
|
31913
|
+
//# sourceMappingURL=chunk-DDBB55L2.js.map
|