@remnic/core 10.0.68 → 10.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +1 -1
- package/dist/{chunk-FAF4A76Q.js → chunk-DDBB55L2.js} +383 -249
- package/dist/chunk-DDBB55L2.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/orchestration/buffer-drain-runner.js +1 -1
- package/dist/orchestration/extraction-persist.d.ts.map +1 -1
- package/dist/orchestration/promotion-dedup-window.d.ts +118 -0
- package/dist/orchestration/promotion-dedup-window.d.ts.map +1 -0
- package/dist/orchestrator.js +1 -1
- package/package.json +2 -2
- package/src/extraction-shared-promotion-parallel-dedup.test.ts +212 -0
- package/src/orchestration/extraction-persist.ts +40 -35
- package/src/orchestration/promotion-dedup-window.ts +319 -0
- package/dist/chunk-FAF4A76Q.js.map +0 -1
package/dist/access-cli.js
CHANGED
|
@@ -5042,221 +5042,6 @@ async function persistUnderFactDedupWindow(storageDir, recheck, write) {
|
|
|
5042
5042
|
);
|
|
5043
5043
|
}
|
|
5044
5044
|
|
|
5045
|
-
// src/orchestration/extraction-injection-gate.ts
|
|
5046
|
-
function serializeInjectionScreenCandidate(candidate) {
|
|
5047
|
-
const body = candidate.category === "procedure" ? buildProcedurePersistBody(candidate.content, candidate.procedureSteps) : candidate.content;
|
|
5048
|
-
const attrs = Object.fromEntries(
|
|
5049
|
-
Object.entries(candidate.structuredAttributes ?? {}).flatMap(([key, value]) => {
|
|
5050
|
-
if (typeof value === "string") return [[key, value]];
|
|
5051
|
-
if (typeof value === "number" || typeof value === "boolean") return [[key, String(value)]];
|
|
5052
|
-
return [];
|
|
5053
|
-
})
|
|
5054
|
-
);
|
|
5055
|
-
return Object.keys(attrs).length > 0 ? `${body}
|
|
5056
|
-
[Attributes: ${normalizeAttributePairs(attrs)}]` : body;
|
|
5057
|
-
}
|
|
5058
|
-
function evaluateInjectionScreen(candidate, enabled, profile = "default") {
|
|
5059
|
-
if (!enabled) return { tags: [] };
|
|
5060
|
-
const content = typeof candidate === "string" ? candidate : serializeInjectionScreenCandidate(candidate);
|
|
5061
|
-
const result = screenCandidateFact(content, profile);
|
|
5062
|
-
return {
|
|
5063
|
-
status: result.quarantine === true ? "pending_review" : void 0,
|
|
5064
|
-
tags: result.quarantine === true ? result.findings.map((finding) => `injection-screen:${finding.rule}`) : []
|
|
5065
|
-
};
|
|
5066
|
-
}
|
|
5067
|
-
function withholdScreenedStrings(values, enabled, profile = "default") {
|
|
5068
|
-
const kept = [];
|
|
5069
|
-
const withheldRules = [];
|
|
5070
|
-
for (const value of values) {
|
|
5071
|
-
if (typeof value !== "string") continue;
|
|
5072
|
-
if (!enabled) {
|
|
5073
|
-
kept.push(value);
|
|
5074
|
-
continue;
|
|
5075
|
-
}
|
|
5076
|
-
const screened = screenCandidateFact(value, profile);
|
|
5077
|
-
if (screened.quarantine) {
|
|
5078
|
-
withheldRules.push(...screened.findings.map((finding) => `injection-screen:${finding.rule}`));
|
|
5079
|
-
} else {
|
|
5080
|
-
kept.push(value);
|
|
5081
|
-
}
|
|
5082
|
-
}
|
|
5083
|
-
return { kept, withheldRules };
|
|
5084
|
-
}
|
|
5085
|
-
function screenEntityForIndex(entity, enabled, profile = "default") {
|
|
5086
|
-
const record = entity;
|
|
5087
|
-
const name = record?.name;
|
|
5088
|
-
const type = record?.type;
|
|
5089
|
-
if (typeof name !== "string" || !name.trim() || typeof type !== "string" || !type.trim()) return null;
|
|
5090
|
-
const nameScreen = withholdScreenedStrings([name], enabled, profile);
|
|
5091
|
-
const factScreen = withholdScreenedStrings(Array.isArray(record?.facts) ? record.facts : [], enabled, profile);
|
|
5092
|
-
const rawSections = Array.isArray(record?.structuredSections) ? record.structuredSections : void 0;
|
|
5093
|
-
const sectionScreens = enabled && rawSections ? rawSections.map((section) => ({ section, screen: withholdScreenedStrings(section?.facts ?? [], true, profile) })) : void 0;
|
|
5094
|
-
return {
|
|
5095
|
-
name,
|
|
5096
|
-
type,
|
|
5097
|
-
source: typeof record?.source === "string" ? record.source : "extraction",
|
|
5098
|
-
facts: factScreen.kept,
|
|
5099
|
-
...sectionScreens ? {
|
|
5100
|
-
structuredSections: sectionScreens.map(({ section, screen }) => {
|
|
5101
|
-
let nextFactIndex = 0;
|
|
5102
|
-
const factOrigins = screen.kept.map((fact) => {
|
|
5103
|
-
const factIndex = section.facts.indexOf(fact, nextFactIndex);
|
|
5104
|
-
nextFactIndex = factIndex + 1;
|
|
5105
|
-
const origin = section.factOrigins?.[factIndex];
|
|
5106
|
-
return origin === void 0 ? void 0 : parseOriginClass(origin);
|
|
5107
|
-
});
|
|
5108
|
-
return {
|
|
5109
|
-
...section,
|
|
5110
|
-
facts: screen.kept,
|
|
5111
|
-
...factOrigins.some((origin) => origin !== void 0) ? { factOrigins } : {}
|
|
5112
|
-
};
|
|
5113
|
-
})
|
|
5114
|
-
} : rawSections ? { structuredSections: rawSections } : {},
|
|
5115
|
-
withheldRules: [
|
|
5116
|
-
...nameScreen.withheldRules,
|
|
5117
|
-
...factScreen.withheldRules,
|
|
5118
|
-
...(sectionScreens ?? []).flatMap(({ screen }) => screen.withheldRules)
|
|
5119
|
-
],
|
|
5120
|
-
withheld: nameScreen.withheldRules.length > 0
|
|
5121
|
-
};
|
|
5122
|
-
}
|
|
5123
|
-
function screenPersistStrings(values, enabled, profile = "default") {
|
|
5124
|
-
const screen = withholdScreenedStrings(values, enabled, profile);
|
|
5125
|
-
return {
|
|
5126
|
-
kept: screen.kept,
|
|
5127
|
-
...screen.withheldRules.length > 0 ? { warning: `injection screen withheld ${values.length - screen.kept.length} candidate(s) [${screen.withheldRules.join(", ")}]` } : {}
|
|
5128
|
-
};
|
|
5129
|
-
}
|
|
5130
|
-
|
|
5131
|
-
// src/orchestration/extraction-envelope.ts
|
|
5132
|
-
function composeSalvagedExtractionEnvelope(input, ctx) {
|
|
5133
|
-
const envelope = composeMemoryEnvelope(input, ctx, { salvage: true });
|
|
5134
|
-
if (envelope.salvageNotes.length > 0) {
|
|
5135
|
-
log.warn(`extraction write salvaged invalid fields: ${envelope.salvageNotes.join("; ")}`);
|
|
5136
|
-
}
|
|
5137
|
-
return envelope;
|
|
5138
|
-
}
|
|
5139
|
-
function withReservedMarkerTag(sourceTags, marker) {
|
|
5140
|
-
const usable = sourceTags.filter(
|
|
5141
|
-
(tag) => typeof tag !== "string" || tag.trim().length <= TAG_LIMITS.maxTagLength
|
|
5142
|
-
);
|
|
5143
|
-
const normalized = normalizeTags(usable) ?? [];
|
|
5144
|
-
const budget = TAG_LIMITS.maxTags - 1;
|
|
5145
|
-
if (normalized.length > budget) {
|
|
5146
|
-
log.warn(
|
|
5147
|
-
`extraction tags exceed ${budget}+marker cap; keeping the first ${budget} of ${normalized.length} and the ${JSON.stringify(marker)} marker`
|
|
5148
|
-
);
|
|
5149
|
-
}
|
|
5150
|
-
const capped = normalized.length > budget ? normalized.slice(0, budget) : normalized;
|
|
5151
|
-
return [...capped, marker];
|
|
5152
|
-
}
|
|
5153
|
-
function probeSalvageSurvivingFields(input) {
|
|
5154
|
-
try {
|
|
5155
|
-
const probe = composeSalvagedExtractionEnvelope(
|
|
5156
|
-
{
|
|
5157
|
-
content: input.content,
|
|
5158
|
-
category: input.category,
|
|
5159
|
-
structuredAttributes: input.structuredAttributes,
|
|
5160
|
-
entityRef: input.entityRef
|
|
5161
|
-
},
|
|
5162
|
-
{ source: "bitemporal-backfill-probe" }
|
|
5163
|
-
);
|
|
5164
|
-
return {
|
|
5165
|
-
entityRef: probe.entityRef,
|
|
5166
|
-
structuredAttributes: probe.rawStructuredAttributes ? { ...probe.rawStructuredAttributes } : void 0
|
|
5167
|
-
};
|
|
5168
|
-
} catch {
|
|
5169
|
-
return null;
|
|
5170
|
-
}
|
|
5171
|
-
}
|
|
5172
|
-
|
|
5173
|
-
// src/dedup/semantic.ts
|
|
5174
|
-
var DEFAULT_SEMANTIC_THRESHOLD = 0.92;
|
|
5175
|
-
var DEFAULT_SEMANTIC_CANDIDATES = 5;
|
|
5176
|
-
function normalizeSemanticThreshold(value) {
|
|
5177
|
-
return Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : DEFAULT_SEMANTIC_THRESHOLD;
|
|
5178
|
-
}
|
|
5179
|
-
function normalizeSemanticCandidates(value) {
|
|
5180
|
-
if (!Number.isFinite(value) || value < 0) {
|
|
5181
|
-
return DEFAULT_SEMANTIC_CANDIDATES;
|
|
5182
|
-
}
|
|
5183
|
-
const normalized = Math.floor(value);
|
|
5184
|
-
return value > 0 && normalized === 0 ? 1 : normalized;
|
|
5185
|
-
}
|
|
5186
|
-
async function decideSemanticDedup(content, lookup, options) {
|
|
5187
|
-
if (!options.enabled) {
|
|
5188
|
-
return { action: "keep", reason: "disabled" };
|
|
5189
|
-
}
|
|
5190
|
-
const threshold = normalizeSemanticThreshold(options.threshold);
|
|
5191
|
-
const candidates = normalizeSemanticCandidates(options.candidates);
|
|
5192
|
-
if (candidates === 0) {
|
|
5193
|
-
return { action: "keep", reason: "disabled" };
|
|
5194
|
-
}
|
|
5195
|
-
const trimmed = typeof content === "string" ? content.trim() : "";
|
|
5196
|
-
if (!trimmed) {
|
|
5197
|
-
return { action: "keep", reason: "no_near_duplicate" };
|
|
5198
|
-
}
|
|
5199
|
-
let hits = [];
|
|
5200
|
-
try {
|
|
5201
|
-
hits = await lookup(trimmed, candidates);
|
|
5202
|
-
} catch {
|
|
5203
|
-
return { action: "keep", reason: "backend_unavailable" };
|
|
5204
|
-
}
|
|
5205
|
-
if (!Array.isArray(hits) || hits.length === 0) {
|
|
5206
|
-
return { action: "keep", reason: "no_candidates" };
|
|
5207
|
-
}
|
|
5208
|
-
const candidateConnector = normalizeConnectorScope(options.sourceConnector);
|
|
5209
|
-
let top;
|
|
5210
|
-
let sameConnectorTop;
|
|
5211
|
-
for (const hit of hits) {
|
|
5212
|
-
if (!hit || typeof hit.id !== "string" || hit.id.trim().length === 0 || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
|
|
5213
|
-
continue;
|
|
5214
|
-
}
|
|
5215
|
-
if (!top || hit.score > top.score) {
|
|
5216
|
-
top = hit;
|
|
5217
|
-
}
|
|
5218
|
-
if (candidateConnector !== void 0 && connectorMatchesScope(hit.sourceConnector, candidateConnector) && (!sameConnectorTop || hit.score > sameConnectorTop.score)) {
|
|
5219
|
-
sameConnectorTop = hit;
|
|
5220
|
-
}
|
|
5221
|
-
}
|
|
5222
|
-
if (!top) {
|
|
5223
|
-
return { action: "keep", reason: "no_near_duplicate" };
|
|
5224
|
-
}
|
|
5225
|
-
if (top.score >= threshold) {
|
|
5226
|
-
if (candidateConnector !== void 0) {
|
|
5227
|
-
const match = sameConnectorTop !== void 0 && sameConnectorTop.score >= threshold ? sameConnectorTop : void 0;
|
|
5228
|
-
if (match) {
|
|
5229
|
-
return {
|
|
5230
|
-
action: "skip",
|
|
5231
|
-
reason: "near_duplicate",
|
|
5232
|
-
topScore: match.score,
|
|
5233
|
-
topId: match.id,
|
|
5234
|
-
topPath: match.path
|
|
5235
|
-
};
|
|
5236
|
-
}
|
|
5237
|
-
return {
|
|
5238
|
-
action: "keep",
|
|
5239
|
-
reason: "connector_mismatch",
|
|
5240
|
-
topScore: top.score,
|
|
5241
|
-
topId: top.id
|
|
5242
|
-
};
|
|
5243
|
-
}
|
|
5244
|
-
return {
|
|
5245
|
-
action: "skip",
|
|
5246
|
-
reason: "near_duplicate",
|
|
5247
|
-
topScore: top.score,
|
|
5248
|
-
topId: top.id,
|
|
5249
|
-
topPath: top.path
|
|
5250
|
-
};
|
|
5251
|
-
}
|
|
5252
|
-
return {
|
|
5253
|
-
action: "keep",
|
|
5254
|
-
reason: "no_near_duplicate",
|
|
5255
|
-
topScore: top.score,
|
|
5256
|
-
topId: top.id
|
|
5257
|
-
};
|
|
5258
|
-
}
|
|
5259
|
-
|
|
5260
5045
|
// src/orchestration/semantic-merge-promotion-payload.ts
|
|
5261
5046
|
import { createHash as createHash3 } from "crypto";
|
|
5262
5047
|
import { readFile as readFile2 } from "fs/promises";
|
|
@@ -5717,6 +5502,356 @@ async function retireStaleMergedTargetPromotionCopies(args) {
|
|
|
5717
5502
|
return retired;
|
|
5718
5503
|
}
|
|
5719
5504
|
|
|
5505
|
+
// src/orchestration/promotion-dedup-window.ts
|
|
5506
|
+
async function findPromotionRaceDuplicate(args) {
|
|
5507
|
+
try {
|
|
5508
|
+
const allMems = await readActiveMemoriesBothTiers(args.storage);
|
|
5509
|
+
const nc = args.connector?.trim() || void 0;
|
|
5510
|
+
const incomingNormalized = ContentHashIndex.normalizeContent(args.dedupContent);
|
|
5511
|
+
return allMems.find((m) => {
|
|
5512
|
+
if (m.frontmatter.category !== args.category) return false;
|
|
5513
|
+
if ((m.frontmatter.status ?? "active") !== "active") return false;
|
|
5514
|
+
if (normalizeStoredHashSource(m.content ?? "", args.citationEnabled, args.citationTemplate) !== incomingNormalized)
|
|
5515
|
+
return false;
|
|
5516
|
+
if (args.entityKeyNorm !== void 0) {
|
|
5517
|
+
if (!m.frontmatter.entityRef) return false;
|
|
5518
|
+
if (normalizeSupersessionKey(m.frontmatter.entityRef) !== args.entityKeyNorm) return false;
|
|
5519
|
+
}
|
|
5520
|
+
return (m.frontmatter.sourceConnector?.trim() || void 0) === nc;
|
|
5521
|
+
});
|
|
5522
|
+
} catch (err) {
|
|
5523
|
+
log.warn(
|
|
5524
|
+
`promotion race recheck scan failed for ${args.storage.dir}; writing fail-open: ${err instanceof Error ? err.message : String(err)}`
|
|
5525
|
+
);
|
|
5526
|
+
return void 0;
|
|
5527
|
+
}
|
|
5528
|
+
}
|
|
5529
|
+
async function confirmPromotionRaceDuplicate(args) {
|
|
5530
|
+
try {
|
|
5531
|
+
if (await args.storage.isFactContentHashAuthoritative()) {
|
|
5532
|
+
if (!await args.storage.hasFactContentHash(args.dedupContent)) return false;
|
|
5533
|
+
}
|
|
5534
|
+
} catch (probeErr) {
|
|
5535
|
+
log.warn(
|
|
5536
|
+
`promotion race recheck authoritative probe failed for ${args.storage.dir}; confirming against corpus: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`
|
|
5537
|
+
);
|
|
5538
|
+
}
|
|
5539
|
+
return await findPromotionRaceDuplicate(args) !== void 0;
|
|
5540
|
+
}
|
|
5541
|
+
function buildPromotionWriteOptions(args) {
|
|
5542
|
+
const { promotion } = args;
|
|
5543
|
+
const writeOptions = {
|
|
5544
|
+
importance: promotion.importance,
|
|
5545
|
+
lineage: [promotion.sourceMemoryId],
|
|
5546
|
+
sourceMemoryId: promotion.sourceMemoryId,
|
|
5547
|
+
intentGoal: promotion.intentGoal,
|
|
5548
|
+
intentActionType: promotion.intentActionType,
|
|
5549
|
+
intentEntityTypes: promotion.intentEntityTypes,
|
|
5550
|
+
memoryKind: promotion.memoryKind,
|
|
5551
|
+
contentHashSource: promotion.category === "fact" ? args.dedupContent : args.rawContent
|
|
5552
|
+
};
|
|
5553
|
+
if (promotion.invalidAt) writeOptions.invalidAt = promotion.invalidAt;
|
|
5554
|
+
if (promotion.observedAt) writeOptions.observedAt = promotion.observedAt;
|
|
5555
|
+
if (promotion.eventTimeSource) writeOptions.eventTimeSource = promotion.eventTimeSource;
|
|
5556
|
+
if (promotion.sources && promotion.sources.length > 0) writeOptions.sources = promotion.sources;
|
|
5557
|
+
if (promotion.provenance) writeOptions.provenance = promotion.provenance;
|
|
5558
|
+
if (args.toolScoped) writeOptions.toolScoped = true;
|
|
5559
|
+
return writeOptions;
|
|
5560
|
+
}
|
|
5561
|
+
async function writePromotionUnderFactDedupWindow(args) {
|
|
5562
|
+
const { destination, envelope, promotion } = args;
|
|
5563
|
+
if (promotion.category !== "fact") {
|
|
5564
|
+
return {
|
|
5565
|
+
suppressed: false,
|
|
5566
|
+
result: await destination.writeSealedMemory(envelope, buildPromotionWriteOptions(args))
|
|
5567
|
+
};
|
|
5568
|
+
}
|
|
5569
|
+
const entityKeyNorm = args.sameEntityGuardEnabled && envelope.entityRef && args.survivingAttrs && Object.keys(args.survivingAttrs).length > 0 ? normalizeSupersessionKey(envelope.entityRef) : void 0;
|
|
5570
|
+
const outcome = await persistUnderFactDedupWindow(
|
|
5571
|
+
destination.dir,
|
|
5572
|
+
() => promotion.category === "fact" ? confirmPromotionRaceDuplicate({
|
|
5573
|
+
storage: destination,
|
|
5574
|
+
dedupContent: args.dedupContent,
|
|
5575
|
+
category: promotion.category,
|
|
5576
|
+
connector: args.connector,
|
|
5577
|
+
citationEnabled: args.citationEnabled,
|
|
5578
|
+
citationTemplate: args.citationTemplate,
|
|
5579
|
+
entityKeyNorm
|
|
5580
|
+
}) : Promise.resolve(false),
|
|
5581
|
+
() => destination.writeSealedMemory(envelope, buildPromotionWriteOptions(args))
|
|
5582
|
+
);
|
|
5583
|
+
if (outcome.suppressed) {
|
|
5584
|
+
log.info(`dedup: raced promotion duplicate suppressed in fact-dedup window for ${destination.dir}`);
|
|
5585
|
+
if (promotion.invalidAt || promotion.validAt) {
|
|
5586
|
+
await args.backfillTemporalBoundsOnDedupHit(
|
|
5587
|
+
destination,
|
|
5588
|
+
args.dedupContent,
|
|
5589
|
+
{
|
|
5590
|
+
invalidAt: promotion.invalidAt,
|
|
5591
|
+
// #1707 thread 2 — carry the corrected start bound.
|
|
5592
|
+
validFrom: promotion.validAt,
|
|
5593
|
+
observedAt: promotion.observedAt,
|
|
5594
|
+
eventTimeSource: promotion.eventTimeSource
|
|
5595
|
+
},
|
|
5596
|
+
envelope.entityRef,
|
|
5597
|
+
args.connector
|
|
5598
|
+
);
|
|
5599
|
+
}
|
|
5600
|
+
if (args.supersession?.supersessionEnabled && promotion.category === "fact" && envelope.entityRef && args.survivingAttrs && Object.keys(args.survivingAttrs).length > 0) {
|
|
5601
|
+
try {
|
|
5602
|
+
const winning = await findPromotionRaceDuplicate({
|
|
5603
|
+
storage: destination,
|
|
5604
|
+
dedupContent: args.dedupContent,
|
|
5605
|
+
category: promotion.category,
|
|
5606
|
+
connector: args.connector,
|
|
5607
|
+
citationEnabled: args.citationEnabled,
|
|
5608
|
+
citationTemplate: args.citationTemplate,
|
|
5609
|
+
// PRRT_kwDORJXyws6z2o11: the winner ANCHORS the pass, so the lookup
|
|
5610
|
+
// is always entity-guarded — without it (profile path) it could
|
|
5611
|
+
// match another entity's equal-content fact and combine that fact's
|
|
5612
|
+
// id with the incoming entity's attributes (PR #402 Uybg hazard).
|
|
5613
|
+
entityKeyNorm: normalizeSupersessionKey(envelope.entityRef)
|
|
5614
|
+
});
|
|
5615
|
+
if (winning) {
|
|
5616
|
+
await applyTemporalSupersession({
|
|
5617
|
+
storage: destination,
|
|
5618
|
+
newMemoryId: winning.frontmatter.id,
|
|
5619
|
+
entityRef: envelope.entityRef,
|
|
5620
|
+
structuredAttributes: { ...args.survivingAttrs },
|
|
5621
|
+
// Ordering anchor mirrors the gate's supersessionOrderingAt: an
|
|
5622
|
+
// incoming event date orders the pass, wall clock otherwise.
|
|
5623
|
+
createdAt: promotion.validAt && promotion.validAt.length > 0 ? promotion.validAt : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5624
|
+
useCallerTimestamp: true,
|
|
5625
|
+
enabled: !(promotion.eventTimeSource === "extracted" && !promotion.validAt),
|
|
5626
|
+
namespaceScope: args.supersession.namespaceScope,
|
|
5627
|
+
dependencyPropagationDelivery: args.supersession.dependencyPropagationDelivery
|
|
5628
|
+
});
|
|
5629
|
+
}
|
|
5630
|
+
} catch (supersessionErr) {
|
|
5631
|
+
log.warn(
|
|
5632
|
+
`promotion window supersession failed open for ${destination.dir}: ${supersessionErr instanceof Error ? supersessionErr.message : String(supersessionErr)}`
|
|
5633
|
+
);
|
|
5634
|
+
}
|
|
5635
|
+
}
|
|
5636
|
+
}
|
|
5637
|
+
return outcome;
|
|
5638
|
+
}
|
|
5639
|
+
|
|
5640
|
+
// src/orchestration/extraction-injection-gate.ts
|
|
5641
|
+
function serializeInjectionScreenCandidate(candidate) {
|
|
5642
|
+
const body = candidate.category === "procedure" ? buildProcedurePersistBody(candidate.content, candidate.procedureSteps) : candidate.content;
|
|
5643
|
+
const attrs = Object.fromEntries(
|
|
5644
|
+
Object.entries(candidate.structuredAttributes ?? {}).flatMap(([key, value]) => {
|
|
5645
|
+
if (typeof value === "string") return [[key, value]];
|
|
5646
|
+
if (typeof value === "number" || typeof value === "boolean") return [[key, String(value)]];
|
|
5647
|
+
return [];
|
|
5648
|
+
})
|
|
5649
|
+
);
|
|
5650
|
+
return Object.keys(attrs).length > 0 ? `${body}
|
|
5651
|
+
[Attributes: ${normalizeAttributePairs(attrs)}]` : body;
|
|
5652
|
+
}
|
|
5653
|
+
function evaluateInjectionScreen(candidate, enabled, profile = "default") {
|
|
5654
|
+
if (!enabled) return { tags: [] };
|
|
5655
|
+
const content = typeof candidate === "string" ? candidate : serializeInjectionScreenCandidate(candidate);
|
|
5656
|
+
const result = screenCandidateFact(content, profile);
|
|
5657
|
+
return {
|
|
5658
|
+
status: result.quarantine === true ? "pending_review" : void 0,
|
|
5659
|
+
tags: result.quarantine === true ? result.findings.map((finding) => `injection-screen:${finding.rule}`) : []
|
|
5660
|
+
};
|
|
5661
|
+
}
|
|
5662
|
+
function withholdScreenedStrings(values, enabled, profile = "default") {
|
|
5663
|
+
const kept = [];
|
|
5664
|
+
const withheldRules = [];
|
|
5665
|
+
for (const value of values) {
|
|
5666
|
+
if (typeof value !== "string") continue;
|
|
5667
|
+
if (!enabled) {
|
|
5668
|
+
kept.push(value);
|
|
5669
|
+
continue;
|
|
5670
|
+
}
|
|
5671
|
+
const screened = screenCandidateFact(value, profile);
|
|
5672
|
+
if (screened.quarantine) {
|
|
5673
|
+
withheldRules.push(...screened.findings.map((finding) => `injection-screen:${finding.rule}`));
|
|
5674
|
+
} else {
|
|
5675
|
+
kept.push(value);
|
|
5676
|
+
}
|
|
5677
|
+
}
|
|
5678
|
+
return { kept, withheldRules };
|
|
5679
|
+
}
|
|
5680
|
+
function screenEntityForIndex(entity, enabled, profile = "default") {
|
|
5681
|
+
const record = entity;
|
|
5682
|
+
const name = record?.name;
|
|
5683
|
+
const type = record?.type;
|
|
5684
|
+
if (typeof name !== "string" || !name.trim() || typeof type !== "string" || !type.trim()) return null;
|
|
5685
|
+
const nameScreen = withholdScreenedStrings([name], enabled, profile);
|
|
5686
|
+
const factScreen = withholdScreenedStrings(Array.isArray(record?.facts) ? record.facts : [], enabled, profile);
|
|
5687
|
+
const rawSections = Array.isArray(record?.structuredSections) ? record.structuredSections : void 0;
|
|
5688
|
+
const sectionScreens = enabled && rawSections ? rawSections.map((section) => ({ section, screen: withholdScreenedStrings(section?.facts ?? [], true, profile) })) : void 0;
|
|
5689
|
+
return {
|
|
5690
|
+
name,
|
|
5691
|
+
type,
|
|
5692
|
+
source: typeof record?.source === "string" ? record.source : "extraction",
|
|
5693
|
+
facts: factScreen.kept,
|
|
5694
|
+
...sectionScreens ? {
|
|
5695
|
+
structuredSections: sectionScreens.map(({ section, screen }) => {
|
|
5696
|
+
let nextFactIndex = 0;
|
|
5697
|
+
const factOrigins = screen.kept.map((fact) => {
|
|
5698
|
+
const factIndex = section.facts.indexOf(fact, nextFactIndex);
|
|
5699
|
+
nextFactIndex = factIndex + 1;
|
|
5700
|
+
const origin = section.factOrigins?.[factIndex];
|
|
5701
|
+
return origin === void 0 ? void 0 : parseOriginClass(origin);
|
|
5702
|
+
});
|
|
5703
|
+
return {
|
|
5704
|
+
...section,
|
|
5705
|
+
facts: screen.kept,
|
|
5706
|
+
...factOrigins.some((origin) => origin !== void 0) ? { factOrigins } : {}
|
|
5707
|
+
};
|
|
5708
|
+
})
|
|
5709
|
+
} : rawSections ? { structuredSections: rawSections } : {},
|
|
5710
|
+
withheldRules: [
|
|
5711
|
+
...nameScreen.withheldRules,
|
|
5712
|
+
...factScreen.withheldRules,
|
|
5713
|
+
...(sectionScreens ?? []).flatMap(({ screen }) => screen.withheldRules)
|
|
5714
|
+
],
|
|
5715
|
+
withheld: nameScreen.withheldRules.length > 0
|
|
5716
|
+
};
|
|
5717
|
+
}
|
|
5718
|
+
function screenPersistStrings(values, enabled, profile = "default") {
|
|
5719
|
+
const screen = withholdScreenedStrings(values, enabled, profile);
|
|
5720
|
+
return {
|
|
5721
|
+
kept: screen.kept,
|
|
5722
|
+
...screen.withheldRules.length > 0 ? { warning: `injection screen withheld ${values.length - screen.kept.length} candidate(s) [${screen.withheldRules.join(", ")}]` } : {}
|
|
5723
|
+
};
|
|
5724
|
+
}
|
|
5725
|
+
|
|
5726
|
+
// src/orchestration/extraction-envelope.ts
|
|
5727
|
+
function composeSalvagedExtractionEnvelope(input, ctx) {
|
|
5728
|
+
const envelope = composeMemoryEnvelope(input, ctx, { salvage: true });
|
|
5729
|
+
if (envelope.salvageNotes.length > 0) {
|
|
5730
|
+
log.warn(`extraction write salvaged invalid fields: ${envelope.salvageNotes.join("; ")}`);
|
|
5731
|
+
}
|
|
5732
|
+
return envelope;
|
|
5733
|
+
}
|
|
5734
|
+
function withReservedMarkerTag(sourceTags, marker) {
|
|
5735
|
+
const usable = sourceTags.filter(
|
|
5736
|
+
(tag) => typeof tag !== "string" || tag.trim().length <= TAG_LIMITS.maxTagLength
|
|
5737
|
+
);
|
|
5738
|
+
const normalized = normalizeTags(usable) ?? [];
|
|
5739
|
+
const budget = TAG_LIMITS.maxTags - 1;
|
|
5740
|
+
if (normalized.length > budget) {
|
|
5741
|
+
log.warn(
|
|
5742
|
+
`extraction tags exceed ${budget}+marker cap; keeping the first ${budget} of ${normalized.length} and the ${JSON.stringify(marker)} marker`
|
|
5743
|
+
);
|
|
5744
|
+
}
|
|
5745
|
+
const capped = normalized.length > budget ? normalized.slice(0, budget) : normalized;
|
|
5746
|
+
return [...capped, marker];
|
|
5747
|
+
}
|
|
5748
|
+
function probeSalvageSurvivingFields(input) {
|
|
5749
|
+
try {
|
|
5750
|
+
const probe = composeSalvagedExtractionEnvelope(
|
|
5751
|
+
{
|
|
5752
|
+
content: input.content,
|
|
5753
|
+
category: input.category,
|
|
5754
|
+
structuredAttributes: input.structuredAttributes,
|
|
5755
|
+
entityRef: input.entityRef
|
|
5756
|
+
},
|
|
5757
|
+
{ source: "bitemporal-backfill-probe" }
|
|
5758
|
+
);
|
|
5759
|
+
return {
|
|
5760
|
+
entityRef: probe.entityRef,
|
|
5761
|
+
structuredAttributes: probe.rawStructuredAttributes ? { ...probe.rawStructuredAttributes } : void 0
|
|
5762
|
+
};
|
|
5763
|
+
} catch {
|
|
5764
|
+
return null;
|
|
5765
|
+
}
|
|
5766
|
+
}
|
|
5767
|
+
|
|
5768
|
+
// src/dedup/semantic.ts
|
|
5769
|
+
var DEFAULT_SEMANTIC_THRESHOLD = 0.92;
|
|
5770
|
+
var DEFAULT_SEMANTIC_CANDIDATES = 5;
|
|
5771
|
+
function normalizeSemanticThreshold(value) {
|
|
5772
|
+
return Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : DEFAULT_SEMANTIC_THRESHOLD;
|
|
5773
|
+
}
|
|
5774
|
+
function normalizeSemanticCandidates(value) {
|
|
5775
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
5776
|
+
return DEFAULT_SEMANTIC_CANDIDATES;
|
|
5777
|
+
}
|
|
5778
|
+
const normalized = Math.floor(value);
|
|
5779
|
+
return value > 0 && normalized === 0 ? 1 : normalized;
|
|
5780
|
+
}
|
|
5781
|
+
async function decideSemanticDedup(content, lookup, options) {
|
|
5782
|
+
if (!options.enabled) {
|
|
5783
|
+
return { action: "keep", reason: "disabled" };
|
|
5784
|
+
}
|
|
5785
|
+
const threshold = normalizeSemanticThreshold(options.threshold);
|
|
5786
|
+
const candidates = normalizeSemanticCandidates(options.candidates);
|
|
5787
|
+
if (candidates === 0) {
|
|
5788
|
+
return { action: "keep", reason: "disabled" };
|
|
5789
|
+
}
|
|
5790
|
+
const trimmed = typeof content === "string" ? content.trim() : "";
|
|
5791
|
+
if (!trimmed) {
|
|
5792
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
5793
|
+
}
|
|
5794
|
+
let hits = [];
|
|
5795
|
+
try {
|
|
5796
|
+
hits = await lookup(trimmed, candidates);
|
|
5797
|
+
} catch {
|
|
5798
|
+
return { action: "keep", reason: "backend_unavailable" };
|
|
5799
|
+
}
|
|
5800
|
+
if (!Array.isArray(hits) || hits.length === 0) {
|
|
5801
|
+
return { action: "keep", reason: "no_candidates" };
|
|
5802
|
+
}
|
|
5803
|
+
const candidateConnector = normalizeConnectorScope(options.sourceConnector);
|
|
5804
|
+
let top;
|
|
5805
|
+
let sameConnectorTop;
|
|
5806
|
+
for (const hit of hits) {
|
|
5807
|
+
if (!hit || typeof hit.id !== "string" || hit.id.trim().length === 0 || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
|
|
5808
|
+
continue;
|
|
5809
|
+
}
|
|
5810
|
+
if (!top || hit.score > top.score) {
|
|
5811
|
+
top = hit;
|
|
5812
|
+
}
|
|
5813
|
+
if (candidateConnector !== void 0 && connectorMatchesScope(hit.sourceConnector, candidateConnector) && (!sameConnectorTop || hit.score > sameConnectorTop.score)) {
|
|
5814
|
+
sameConnectorTop = hit;
|
|
5815
|
+
}
|
|
5816
|
+
}
|
|
5817
|
+
if (!top) {
|
|
5818
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
5819
|
+
}
|
|
5820
|
+
if (top.score >= threshold) {
|
|
5821
|
+
if (candidateConnector !== void 0) {
|
|
5822
|
+
const match = sameConnectorTop !== void 0 && sameConnectorTop.score >= threshold ? sameConnectorTop : void 0;
|
|
5823
|
+
if (match) {
|
|
5824
|
+
return {
|
|
5825
|
+
action: "skip",
|
|
5826
|
+
reason: "near_duplicate",
|
|
5827
|
+
topScore: match.score,
|
|
5828
|
+
topId: match.id,
|
|
5829
|
+
topPath: match.path
|
|
5830
|
+
};
|
|
5831
|
+
}
|
|
5832
|
+
return {
|
|
5833
|
+
action: "keep",
|
|
5834
|
+
reason: "connector_mismatch",
|
|
5835
|
+
topScore: top.score,
|
|
5836
|
+
topId: top.id
|
|
5837
|
+
};
|
|
5838
|
+
}
|
|
5839
|
+
return {
|
|
5840
|
+
action: "skip",
|
|
5841
|
+
reason: "near_duplicate",
|
|
5842
|
+
topScore: top.score,
|
|
5843
|
+
topId: top.id,
|
|
5844
|
+
topPath: top.path
|
|
5845
|
+
};
|
|
5846
|
+
}
|
|
5847
|
+
return {
|
|
5848
|
+
action: "keep",
|
|
5849
|
+
reason: "no_near_duplicate",
|
|
5850
|
+
topScore: top.score,
|
|
5851
|
+
topId: top.id
|
|
5852
|
+
};
|
|
5853
|
+
}
|
|
5854
|
+
|
|
5720
5855
|
// src/orchestration/semantic-merge-persist.ts
|
|
5721
5856
|
import path6 from "path";
|
|
5722
5857
|
|
|
@@ -6736,23 +6871,23 @@ var ExtractionPersistCoordinator = class {
|
|
|
6736
6871
|
continue;
|
|
6737
6872
|
}
|
|
6738
6873
|
}
|
|
6739
|
-
const
|
|
6740
|
-
|
|
6741
|
-
|
|
6742
|
-
|
|
6743
|
-
|
|
6744
|
-
|
|
6745
|
-
|
|
6746
|
-
|
|
6747
|
-
|
|
6748
|
-
|
|
6749
|
-
|
|
6750
|
-
|
|
6751
|
-
|
|
6752
|
-
|
|
6753
|
-
...options.provenance ? { provenance: options.provenance } : {},
|
|
6754
|
-
...options.toolScoped ? { toolScoped: true } : {}
|
|
6874
|
+
const promotionWrite = await writePromotionUnderFactDedupWindow({
|
|
6875
|
+
destination: targetStorage,
|
|
6876
|
+
envelope: targetPromotionEnvelope,
|
|
6877
|
+
promotion: options,
|
|
6878
|
+
dedupContent,
|
|
6879
|
+
rawContent,
|
|
6880
|
+
toolScoped: options.toolScoped === true,
|
|
6881
|
+
connector: sourceContext?.sourceConnector,
|
|
6882
|
+
citationEnabled,
|
|
6883
|
+
citationTemplate,
|
|
6884
|
+
sameEntityGuardEnabled: false,
|
|
6885
|
+
survivingAttrs: targetSurvivingAttrs,
|
|
6886
|
+
supersession: { supersessionEnabled: lifecycleCaps.temporalSupersession, dependencyPropagationDelivery: this.dependencyPropagationDelivery, namespaceScope: this.deps.storageDirNamespace(targetStorage.dir) },
|
|
6887
|
+
backfillTemporalBoundsOnDedupHit: this.deps.backfillTemporalBoundsOnDedupHit
|
|
6755
6888
|
});
|
|
6889
|
+
if (promotionWrite.suppressed) continue;
|
|
6890
|
+
const targetPromotion = promotionWrite.result;
|
|
6756
6891
|
const promotedId = targetPromotion.id;
|
|
6757
6892
|
if (!targetPromotion.tombstoneBlocked && lifecycleCaps.temporalSupersession && options.category === "fact" && targetPromotionEnvelope.entityRef && targetSurvivingAttrs && Object.keys(targetSurvivingAttrs).length > 0) {
|
|
6758
6893
|
try {
|
|
@@ -6956,24 +7091,23 @@ var ExtractionPersistCoordinator = class {
|
|
|
6956
7091
|
}
|
|
6957
7092
|
}
|
|
6958
7093
|
}
|
|
6959
|
-
const
|
|
6960
|
-
|
|
6961
|
-
|
|
6962
|
-
|
|
6963
|
-
|
|
6964
|
-
|
|
6965
|
-
|
|
6966
|
-
|
|
6967
|
-
|
|
6968
|
-
|
|
6969
|
-
|
|
6970
|
-
|
|
6971
|
-
|
|
6972
|
-
|
|
6973
|
-
...options.sources && options.sources.length > 0 ? { sources: options.sources } : {},
|
|
6974
|
-
...options.provenance ? { provenance: options.provenance } : {},
|
|
6975
|
-
...toolScoped ? { toolScoped: true } : {}
|
|
7094
|
+
const sharedWrite = await writePromotionUnderFactDedupWindow({
|
|
7095
|
+
destination: sharedStorage,
|
|
7096
|
+
envelope: sharedPromotionEnvelope,
|
|
7097
|
+
promotion: options,
|
|
7098
|
+
dedupContent,
|
|
7099
|
+
rawContent,
|
|
7100
|
+
toolScoped,
|
|
7101
|
+
connector: sourceContext?.sourceConnector,
|
|
7102
|
+
citationEnabled,
|
|
7103
|
+
citationTemplate,
|
|
7104
|
+
sameEntityGuardEnabled: lifecycleCaps.temporalSupersession,
|
|
7105
|
+
survivingAttrs: sharedSurvivingAttrs,
|
|
7106
|
+
supersession: { supersessionEnabled: lifecycleCaps.temporalSupersession, dependencyPropagationDelivery: this.dependencyPropagationDelivery, namespaceScope: this.deps.storageDirNamespace(sharedStorage.dir) },
|
|
7107
|
+
backfillTemporalBoundsOnDedupHit: this.deps.backfillTemporalBoundsOnDedupHit
|
|
6976
7108
|
});
|
|
7109
|
+
if (sharedWrite.suppressed) return promotedCopyId;
|
|
7110
|
+
const sharedPromotion = sharedWrite.result;
|
|
6977
7111
|
const promotedId = sharedPromotion.id;
|
|
6978
7112
|
if (!sharedPromotion.tombstoneBlocked && lifecycleCaps.temporalSupersession && sharedPromotionEnvelope.entityRef && sharedSurvivingAttrs && Object.keys(sharedSurvivingAttrs).length > 0) {
|
|
6979
7113
|
try {
|
|
@@ -31776,4 +31910,4 @@ export {
|
|
|
31776
31910
|
qmdStartupCollectionChecksWithTimeout,
|
|
31777
31911
|
Orchestrator
|
|
31778
31912
|
};
|
|
31779
|
-
//# sourceMappingURL=chunk-
|
|
31913
|
+
//# sourceMappingURL=chunk-DDBB55L2.js.map
|