@inerrata-corporation/errata 2.0.2-dev.477 → 2.0.2-dev.483
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/consolidate-worker.mjs +5 -2
- package/errata.mjs +101 -16
- package/package.json +1 -1
- package/pass-worker.mjs +22 -7
package/consolidate-worker.mjs
CHANGED
|
@@ -673,7 +673,7 @@ function lemma(token) {
|
|
|
673
673
|
}
|
|
674
674
|
return token;
|
|
675
675
|
}
|
|
676
|
-
function
|
|
676
|
+
function tokenize(text, resolve) {
|
|
677
677
|
const matches = text.normalize("NFC").toLowerCase().match(TOKEN_RE) ?? [];
|
|
678
678
|
const out = /* @__PURE__ */ new Set();
|
|
679
679
|
for (const raw of matches) {
|
|
@@ -683,11 +683,14 @@ function conceptTokens(text) {
|
|
|
683
683
|
out.add(token);
|
|
684
684
|
continue;
|
|
685
685
|
}
|
|
686
|
-
const canonical =
|
|
686
|
+
const canonical = resolve(token);
|
|
687
687
|
out.add(canonical ?? lemma(token));
|
|
688
688
|
}
|
|
689
689
|
return [...out].sort();
|
|
690
690
|
}
|
|
691
|
+
function conceptTokens(text) {
|
|
692
|
+
return tokenize(text, resolveCanonicalId);
|
|
693
|
+
}
|
|
691
694
|
function conceptBag(text) {
|
|
692
695
|
return conceptTokens(text).join(" ");
|
|
693
696
|
}
|
package/errata.mjs
CHANGED
|
@@ -619,6 +619,11 @@ function resolveCanonical(label) {
|
|
|
619
619
|
function resolveCanonicalId(label) {
|
|
620
620
|
return resolveCanonical(label)?.id;
|
|
621
621
|
}
|
|
622
|
+
function resolveUnambiguousCanonicalId(label) {
|
|
623
|
+
const key = label.trim().toLowerCase();
|
|
624
|
+
if (AMBIGUOUS_ALIASES.has(key)) return void 0;
|
|
625
|
+
return resolveCanonicalId(key);
|
|
626
|
+
}
|
|
622
627
|
function allEntities() {
|
|
623
628
|
return REGISTRY;
|
|
624
629
|
}
|
|
@@ -628,7 +633,7 @@ function aliasesLongestFirst() {
|
|
|
628
633
|
out2.sort((x, y) => y.alias.length - x.alias.length);
|
|
629
634
|
return out2;
|
|
630
635
|
}
|
|
631
|
-
var REGISTRY, BY_ALIAS;
|
|
636
|
+
var REGISTRY, AMBIGUOUS_ALIASES, BY_ALIAS;
|
|
632
637
|
var init_taxonomy = __esm({
|
|
633
638
|
"../../packages/shared/src/nlp/taxonomy.ts"() {
|
|
634
639
|
"use strict";
|
|
@@ -677,6 +682,13 @@ var init_taxonomy = __esm({
|
|
|
677
682
|
{ id: "concept:retry", name: "retry", category: "concept", aliases: ["retry"] },
|
|
678
683
|
{ id: "concept:migration", name: "migration", category: "concept", aliases: ["migration"] }
|
|
679
684
|
];
|
|
685
|
+
AMBIGUOUS_ALIASES = /* @__PURE__ */ new Set([
|
|
686
|
+
"node",
|
|
687
|
+
"go",
|
|
688
|
+
"pool",
|
|
689
|
+
"spring",
|
|
690
|
+
"ws"
|
|
691
|
+
]);
|
|
680
692
|
BY_ALIAS = /* @__PURE__ */ new Map();
|
|
681
693
|
for (const e of REGISTRY) {
|
|
682
694
|
for (const a of e.aliases) {
|
|
@@ -785,7 +797,7 @@ function lemma(token) {
|
|
|
785
797
|
}
|
|
786
798
|
return token;
|
|
787
799
|
}
|
|
788
|
-
function
|
|
800
|
+
function tokenize(text, resolve6) {
|
|
789
801
|
const matches = text.normalize("NFC").toLowerCase().match(TOKEN_RE) ?? [];
|
|
790
802
|
const out2 = /* @__PURE__ */ new Set();
|
|
791
803
|
for (const raw2 of matches) {
|
|
@@ -795,11 +807,17 @@ function conceptTokens(text) {
|
|
|
795
807
|
out2.add(token);
|
|
796
808
|
continue;
|
|
797
809
|
}
|
|
798
|
-
const canonical =
|
|
810
|
+
const canonical = resolve6(token);
|
|
799
811
|
out2.add(canonical ?? lemma(token));
|
|
800
812
|
}
|
|
801
813
|
return [...out2].sort();
|
|
802
814
|
}
|
|
815
|
+
function conceptTokens(text) {
|
|
816
|
+
return tokenize(text, resolveCanonicalId);
|
|
817
|
+
}
|
|
818
|
+
function retrievalTokens(text) {
|
|
819
|
+
return tokenize(text, resolveUnambiguousCanonicalId);
|
|
820
|
+
}
|
|
803
821
|
function conceptBag(text) {
|
|
804
822
|
return conceptTokens(text).join(" ");
|
|
805
823
|
}
|
|
@@ -15935,6 +15953,7 @@ __export(src_exports, {
|
|
|
15935
15953
|
ABSTRACTION_LEVEL: () => ABSTRACTION_LEVEL,
|
|
15936
15954
|
ALL_EDGE_TYPES: () => ALL_EDGE_TYPES,
|
|
15937
15955
|
ALL_NODE_LABELS: () => ALL_NODE_LABELS,
|
|
15956
|
+
AMBIGUOUS_ALIASES: () => AMBIGUOUS_ALIASES,
|
|
15938
15957
|
BELIEF_EDGES: () => BELIEF_EDGES,
|
|
15939
15958
|
BRIDGE_EDGES: () => BRIDGE_EDGES,
|
|
15940
15959
|
CAUSAL_EDGES: () => CAUSAL_EDGES,
|
|
@@ -16035,6 +16054,8 @@ __export(src_exports, {
|
|
|
16035
16054
|
provenanceBand: () => provenanceBand,
|
|
16036
16055
|
resolveCanonical: () => resolveCanonical,
|
|
16037
16056
|
resolveCanonicalId: () => resolveCanonicalId,
|
|
16057
|
+
resolveUnambiguousCanonicalId: () => resolveUnambiguousCanonicalId,
|
|
16058
|
+
retrievalTokens: () => retrievalTokens,
|
|
16038
16059
|
sha256: () => sha256,
|
|
16039
16060
|
sha256Short: () => sha256Short,
|
|
16040
16061
|
splitPnpmKey: () => splitPnpmKey,
|
|
@@ -18192,7 +18213,7 @@ function djb2Signed(s) {
|
|
|
18192
18213
|
}
|
|
18193
18214
|
return h;
|
|
18194
18215
|
}
|
|
18195
|
-
function
|
|
18216
|
+
function tokenize2(text) {
|
|
18196
18217
|
return text.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0 && t.length <= 40);
|
|
18197
18218
|
}
|
|
18198
18219
|
function* charTrigrams(text) {
|
|
@@ -18204,7 +18225,7 @@ function* charTrigrams(text) {
|
|
|
18204
18225
|
function embed(text) {
|
|
18205
18226
|
const vec = new Array(EMBEDDING_DIM).fill(0);
|
|
18206
18227
|
if (!text) return vec;
|
|
18207
|
-
for (const tok of
|
|
18228
|
+
for (const tok of tokenize2(text)) {
|
|
18208
18229
|
const idx = djb2(tok) % EMBEDDING_DIM;
|
|
18209
18230
|
const sign = djb2Signed(tok) & 1 ? 1 : -1;
|
|
18210
18231
|
vec[idx] = vec[idx] + sign;
|
|
@@ -18578,7 +18599,7 @@ function mergeDuplicateProblems(store, opts) {
|
|
|
18578
18599
|
const report = { clusters: 0, merged: 0 };
|
|
18579
18600
|
const open2 = store.findNodesByLabel("Problem").filter((p) => p.attrs["resolvedAt"] == null && p.attrs["mergedInto"] === void 0);
|
|
18580
18601
|
const tokens = /* @__PURE__ */ new Map();
|
|
18581
|
-
for (const p of open2) tokens.set(p.id, new Set(
|
|
18602
|
+
for (const p of open2) tokens.set(p.id, new Set(retrievalTokens(p.description)));
|
|
18582
18603
|
const consumed = /* @__PURE__ */ new Set();
|
|
18583
18604
|
store.transaction(() => {
|
|
18584
18605
|
for (let i2 = 0; i2 < open2.length; i2++) {
|
|
@@ -18656,12 +18677,12 @@ function findMintTimeDuplicate(store, statement, kind, opts) {
|
|
|
18656
18677
|
if (isConstraintProblem(p) !== incomingIsConstraint) continue;
|
|
18657
18678
|
if (statementIdentityKey(p.description) === key) return p.id;
|
|
18658
18679
|
}
|
|
18659
|
-
const ta = new Set(
|
|
18680
|
+
const ta = new Set(retrievalTokens(statement));
|
|
18660
18681
|
if (ta.size < minTokens) return null;
|
|
18661
18682
|
let best = null;
|
|
18662
18683
|
for (const p of open2) {
|
|
18663
18684
|
if (isConstraintProblem(p) !== incomingIsConstraint) continue;
|
|
18664
|
-
const tb = new Set(
|
|
18685
|
+
const tb = new Set(retrievalTokens(p.description));
|
|
18665
18686
|
if (Math.min(ta.size, tb.size) < minTokens) continue;
|
|
18666
18687
|
const score2 = overlap(ta, tb);
|
|
18667
18688
|
if (score2 >= minOverlap && (best === null || score2 > best.score)) {
|
|
@@ -19368,7 +19389,7 @@ function parseIntentFences(text) {
|
|
|
19368
19389
|
function searchByIntent(store, intent, opts) {
|
|
19369
19390
|
const limit = opts?.limit ?? 5;
|
|
19370
19391
|
const minCoverage = opts?.minCoverage ?? 0.25;
|
|
19371
|
-
const want = new Set(
|
|
19392
|
+
const want = new Set(retrievalTokens(intent));
|
|
19372
19393
|
if (want.size < MIN_INTENT_TOKENS) return [];
|
|
19373
19394
|
const hits = [];
|
|
19374
19395
|
for (const label of RECALLABLE) {
|
|
@@ -19377,7 +19398,7 @@ function searchByIntent(store, intent, opts) {
|
|
|
19377
19398
|
if (label === "Problem" && node2.attrs["resolvedAt"] != null && !isConstraintProblem(node2)) {
|
|
19378
19399
|
continue;
|
|
19379
19400
|
}
|
|
19380
|
-
const bag =
|
|
19401
|
+
const bag = retrievalTokens(node2.description ?? "");
|
|
19381
19402
|
if (bag.length === 0) continue;
|
|
19382
19403
|
const matched = bag.filter((t) => want.has(t));
|
|
19383
19404
|
if (matched.length === 0) continue;
|
|
@@ -22114,7 +22135,24 @@ function renderSnapshot(s) {
|
|
|
22114
22135
|
}
|
|
22115
22136
|
lines.push("");
|
|
22116
22137
|
}
|
|
22117
|
-
if (s.
|
|
22138
|
+
if (s.statedIntent) {
|
|
22139
|
+
const si = s.statedIntent;
|
|
22140
|
+
lines.push(`### \u{1F3AF} You said you're working on this`);
|
|
22141
|
+
lines.push(`> ${si.text}`);
|
|
22142
|
+
if (si.hits.length === 0) {
|
|
22143
|
+
lines.push("");
|
|
22144
|
+
lines.push(
|
|
22145
|
+
"_Nothing recorded here matches that yet \u2014 you're first. What you hit and how you resolve it is what a later agent will be shown._"
|
|
22146
|
+
);
|
|
22147
|
+
} else {
|
|
22148
|
+
lines.push("");
|
|
22149
|
+
lines.push("_Priors that speak to it \u2014 cite one with its `[handle]` if you lean on it:_");
|
|
22150
|
+
for (const h of si.hits) {
|
|
22151
|
+
lines.push(`- **${h.description}** \u2014 \`${h.id}\``);
|
|
22152
|
+
}
|
|
22153
|
+
}
|
|
22154
|
+
lines.push("");
|
|
22155
|
+
} else if (s.workingFile) {
|
|
22118
22156
|
const w = s.workingFile;
|
|
22119
22157
|
lines.push(`### Working file: \`${w.relPath}\``);
|
|
22120
22158
|
if (w.symbols.length) {
|
|
@@ -22227,6 +22265,7 @@ function dropLowestUnit(s) {
|
|
|
22227
22265
|
if (s.motifs.length) return s.motifs.pop(), true;
|
|
22228
22266
|
break;
|
|
22229
22267
|
case "workingFile":
|
|
22268
|
+
if (s.statedIntent) return delete s.statedIntent, true;
|
|
22230
22269
|
if (s.workingFile) return delete s.workingFile, true;
|
|
22231
22270
|
break;
|
|
22232
22271
|
case "remoteOverFloor":
|
|
@@ -22275,6 +22314,7 @@ function assembleAgentContext(opts) {
|
|
|
22275
22314
|
if (opts.edgeElicitation) snapshot.edgeElicitation = opts.edgeElicitation;
|
|
22276
22315
|
if (opts.pendingUpdate) snapshot.pendingUpdate = opts.pendingUpdate;
|
|
22277
22316
|
if (opts.reviewItems?.length) snapshot.reviewItems = [...opts.reviewItems];
|
|
22317
|
+
if (opts.statedIntent) snapshot.statedIntent = opts.statedIntent;
|
|
22278
22318
|
if (opts.remote && opts.remote.length > 0) {
|
|
22279
22319
|
const seen = /* @__PURE__ */ new Set();
|
|
22280
22320
|
const remote = opts.remote.filter((n) => {
|
|
@@ -22315,6 +22355,7 @@ function snapshotImpressions(s) {
|
|
|
22315
22355
|
(s.reviewItems ?? []).map((r) => r.kind === "abstraction" ? r.candidate : r.route)
|
|
22316
22356
|
);
|
|
22317
22357
|
push("workingFile", s.workingFile?.priors?.openProblems.map((p) => p.id) ?? []);
|
|
22358
|
+
push("statedIntent", (s.statedIntent?.hits ?? []).map((h) => h.id));
|
|
22318
22359
|
return out2;
|
|
22319
22360
|
}
|
|
22320
22361
|
function diffRenderLedger(before, after) {
|
|
@@ -29012,7 +29053,7 @@ function popcount(x) {
|
|
|
29012
29053
|
}
|
|
29013
29054
|
return c;
|
|
29014
29055
|
}
|
|
29015
|
-
function
|
|
29056
|
+
function tokenize3(text) {
|
|
29016
29057
|
return text.match(/[A-Za-z_$][A-Za-z0-9_$]*|\d+|[^\s\w]/g) ?? [];
|
|
29017
29058
|
}
|
|
29018
29059
|
function shingle(tokens, n = SHINGLE_N) {
|
|
@@ -29041,7 +29082,7 @@ function simhashFeatures(features) {
|
|
|
29041
29082
|
return out2;
|
|
29042
29083
|
}
|
|
29043
29084
|
function simhash(text) {
|
|
29044
|
-
return simhashFeatures(shingle(
|
|
29085
|
+
return simhashFeatures(shingle(tokenize3(text)));
|
|
29045
29086
|
}
|
|
29046
29087
|
function hammingDistance(a, b) {
|
|
29047
29088
|
return popcount((a ^ b) & MASK64);
|
|
@@ -37564,7 +37605,7 @@ __export(src_exports5, {
|
|
|
37564
37605
|
simhashFeatures: () => simhashFeatures,
|
|
37565
37606
|
symbolNodeId: () => symbolNodeId,
|
|
37566
37607
|
toHex: () => toHex,
|
|
37567
|
-
tokenize: () =>
|
|
37608
|
+
tokenize: () => tokenize3
|
|
37568
37609
|
});
|
|
37569
37610
|
function defaultProviders() {
|
|
37570
37611
|
if (providersCache) return providersCache;
|
|
@@ -53114,6 +53155,38 @@ ${GITIGNORE_LINES.join("\n")}
|
|
|
53114
53155
|
}
|
|
53115
53156
|
}
|
|
53116
53157
|
|
|
53158
|
+
// src/intent-state.ts
|
|
53159
|
+
var INTENT_TTL_MS = 15 * 60 * 1e3;
|
|
53160
|
+
function createIntentState(ttlMs = INTENT_TTL_MS) {
|
|
53161
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
53162
|
+
let recent = null;
|
|
53163
|
+
const live = (s, nowMs) => s !== void 0 && nowMs - s.atMs < ttlMs;
|
|
53164
|
+
return {
|
|
53165
|
+
set(sessionId, text, nowMs) {
|
|
53166
|
+
const trimmed = text.trim();
|
|
53167
|
+
if (!trimmed) return false;
|
|
53168
|
+
const prev = bySession.get(sessionId);
|
|
53169
|
+
bySession.set(sessionId, { text: trimmed, atMs: nowMs });
|
|
53170
|
+
const changed = !(recent && recent.text === trimmed);
|
|
53171
|
+
recent = { sessionId, text: trimmed, atMs: nowMs };
|
|
53172
|
+
if (!changed && prev && prev.text === trimmed) return false;
|
|
53173
|
+
return changed;
|
|
53174
|
+
},
|
|
53175
|
+
get(sessionId, nowMs) {
|
|
53176
|
+
const s = bySession.get(sessionId);
|
|
53177
|
+
return live(s, nowMs) ? s.text : void 0;
|
|
53178
|
+
},
|
|
53179
|
+
mostRecent(nowMs) {
|
|
53180
|
+
if (!recent) return void 0;
|
|
53181
|
+
return nowMs - recent.atMs < ttlMs ? recent.text : void 0;
|
|
53182
|
+
},
|
|
53183
|
+
clear(sessionId) {
|
|
53184
|
+
bySession.delete(sessionId);
|
|
53185
|
+
if (recent?.sessionId === sessionId) recent = null;
|
|
53186
|
+
}
|
|
53187
|
+
};
|
|
53188
|
+
}
|
|
53189
|
+
|
|
53117
53190
|
// src/working-file-state.ts
|
|
53118
53191
|
function createWorkingFileState() {
|
|
53119
53192
|
const bySession = /* @__PURE__ */ new Map();
|
|
@@ -53631,7 +53704,7 @@ function startLoopLagMonitor(thresholdMs = 1e3) {
|
|
|
53631
53704
|
}
|
|
53632
53705
|
|
|
53633
53706
|
// src/engine.ts
|
|
53634
|
-
var DAEMON_VERSION = true ? "2.0.2-dev.
|
|
53707
|
+
var DAEMON_VERSION = true ? "2.0.2-dev.483" : "2.0.0-alpha.0";
|
|
53635
53708
|
var IGNORED_PATH = /[\\/](?:\.git|node_modules|\.errata|\.claude|\.codex|\.cursor|\.turbo|\.next|dist|coverage|test-results|playwright-report|__pycache__)(?:[\\/]|$)/;
|
|
53636
53709
|
var IGNORED_NOISE = /castalia\.db|eventlog\.sqlite|turn-cursors/;
|
|
53637
53710
|
var GIT_OP_MUTE_MS = 4e3;
|
|
@@ -53935,6 +54008,7 @@ function createWorkspaceEngine(opts) {
|
|
|
53935
54008
|
});
|
|
53936
54009
|
}
|
|
53937
54010
|
const workingFiles = createWorkingFileState();
|
|
54011
|
+
const intents = createIntentState();
|
|
53938
54012
|
const toolRuns = createToolRunState();
|
|
53939
54013
|
let ctxRefreshTimer = null;
|
|
53940
54014
|
let contextDirty = false;
|
|
@@ -54002,6 +54076,13 @@ function createWorkspaceEngine(opts) {
|
|
|
54002
54076
|
const doneRender = prebuilt ? null : markPass(`context-render-inline:${profile.name}`);
|
|
54003
54077
|
const pendingUpdate = opts.pendingUpdate?.() ?? null;
|
|
54004
54078
|
const reviewItems = pickReviewItems();
|
|
54079
|
+
const liveIntent = intents.mostRecent(Date.now());
|
|
54080
|
+
const intentHits = liveIntent ? searchByIntent(store, liveIntent, { limit: 4 }).map((h) => ({
|
|
54081
|
+
id: h.node.id,
|
|
54082
|
+
label: h.node.label,
|
|
54083
|
+
description: h.node.description,
|
|
54084
|
+
coverage: h.coverage
|
|
54085
|
+
})) : [];
|
|
54005
54086
|
const { body: body2, snapshot, ledger } = assembleAgentContext({
|
|
54006
54087
|
store,
|
|
54007
54088
|
...snapOpts,
|
|
@@ -54009,7 +54090,8 @@ function createWorkspaceEngine(opts) {
|
|
|
54009
54090
|
...elicit ? { edgeElicitation: { instruction: PRIOR_TAG_INSTRUCTION } } : {},
|
|
54010
54091
|
...prebuilt ? { snapshot: prebuilt } : {},
|
|
54011
54092
|
...pendingUpdate ? { pendingUpdate } : {},
|
|
54012
|
-
...reviewItems.length ? { reviewItems } : {}
|
|
54093
|
+
...reviewItems.length ? { reviewItems } : {},
|
|
54094
|
+
...liveIntent ? { statedIntent: { text: liveIntent, hits: intentHits } } : {}
|
|
54013
54095
|
});
|
|
54014
54096
|
doneRender?.();
|
|
54015
54097
|
try {
|
|
@@ -54371,6 +54453,9 @@ function createWorkspaceEngine(opts) {
|
|
|
54371
54453
|
const touchedFileId = turnFile ? resolveFileId(turnFile) : void 0;
|
|
54372
54454
|
const touched = new Set(toolRuns.get(sessionId));
|
|
54373
54455
|
if (touchedFileId) touched.add(touchedFileId);
|
|
54456
|
+
for (const fence of parseIntentFences(turn.text)) {
|
|
54457
|
+
if (intents.set(sessionId, fence.intent, t)) refreshContextNow();
|
|
54458
|
+
}
|
|
54374
54459
|
const plan = harvestInlineTags(store, turn.text, {
|
|
54375
54460
|
sourceId: sessionLastProblem.get(sessionId),
|
|
54376
54461
|
handleMap,
|
package/package.json
CHANGED
package/pass-worker.mjs
CHANGED
|
@@ -458,13 +458,18 @@ function resolveCanonical(label) {
|
|
|
458
458
|
function resolveCanonicalId(label) {
|
|
459
459
|
return resolveCanonical(label)?.id;
|
|
460
460
|
}
|
|
461
|
+
function resolveUnambiguousCanonicalId(label) {
|
|
462
|
+
const key = label.trim().toLowerCase();
|
|
463
|
+
if (AMBIGUOUS_ALIASES.has(key)) return void 0;
|
|
464
|
+
return resolveCanonicalId(key);
|
|
465
|
+
}
|
|
461
466
|
function aliasesLongestFirst() {
|
|
462
467
|
const out2 = [];
|
|
463
468
|
for (const e of REGISTRY) for (const a of e.aliases) out2.push({ alias: a.toLowerCase(), entity: e });
|
|
464
469
|
out2.sort((x, y) => y.alias.length - x.alias.length);
|
|
465
470
|
return out2;
|
|
466
471
|
}
|
|
467
|
-
var REGISTRY, BY_ALIAS;
|
|
472
|
+
var REGISTRY, AMBIGUOUS_ALIASES, BY_ALIAS;
|
|
468
473
|
var init_taxonomy = __esm({
|
|
469
474
|
"../../packages/shared/src/nlp/taxonomy.ts"() {
|
|
470
475
|
"use strict";
|
|
@@ -513,6 +518,13 @@ var init_taxonomy = __esm({
|
|
|
513
518
|
{ id: "concept:retry", name: "retry", category: "concept", aliases: ["retry"] },
|
|
514
519
|
{ id: "concept:migration", name: "migration", category: "concept", aliases: ["migration"] }
|
|
515
520
|
];
|
|
521
|
+
AMBIGUOUS_ALIASES = /* @__PURE__ */ new Set([
|
|
522
|
+
"node",
|
|
523
|
+
"go",
|
|
524
|
+
"pool",
|
|
525
|
+
"spring",
|
|
526
|
+
"ws"
|
|
527
|
+
]);
|
|
516
528
|
BY_ALIAS = /* @__PURE__ */ new Map();
|
|
517
529
|
for (const e of REGISTRY) {
|
|
518
530
|
for (const a of e.aliases) {
|
|
@@ -574,7 +586,7 @@ function lemma(token) {
|
|
|
574
586
|
}
|
|
575
587
|
return token;
|
|
576
588
|
}
|
|
577
|
-
function
|
|
589
|
+
function tokenize(text, resolve) {
|
|
578
590
|
const matches = text.normalize("NFC").toLowerCase().match(TOKEN_RE) ?? [];
|
|
579
591
|
const out2 = /* @__PURE__ */ new Set();
|
|
580
592
|
for (const raw of matches) {
|
|
@@ -584,11 +596,14 @@ function conceptTokens(text) {
|
|
|
584
596
|
out2.add(token);
|
|
585
597
|
continue;
|
|
586
598
|
}
|
|
587
|
-
const canonical =
|
|
599
|
+
const canonical = resolve(token);
|
|
588
600
|
out2.add(canonical ?? lemma(token));
|
|
589
601
|
}
|
|
590
602
|
return [...out2].sort();
|
|
591
603
|
}
|
|
604
|
+
function retrievalTokens(text) {
|
|
605
|
+
return tokenize(text, resolveUnambiguousCanonicalId);
|
|
606
|
+
}
|
|
592
607
|
var STOPWORDS, NEGATION_TOKENS, TOKEN_RE;
|
|
593
608
|
var init_concept_bag = __esm({
|
|
594
609
|
"../../packages/shared/src/nlp/concept-bag.ts"() {
|
|
@@ -15533,7 +15548,7 @@ function popcount(x) {
|
|
|
15533
15548
|
}
|
|
15534
15549
|
return c;
|
|
15535
15550
|
}
|
|
15536
|
-
function
|
|
15551
|
+
function tokenize2(text) {
|
|
15537
15552
|
return text.match(/[A-Za-z_$][A-Za-z0-9_$]*|\d+|[^\s\w]/g) ?? [];
|
|
15538
15553
|
}
|
|
15539
15554
|
function shingle(tokens, n = SHINGLE_N) {
|
|
@@ -15562,7 +15577,7 @@ function simhashFeatures(features) {
|
|
|
15562
15577
|
return out2;
|
|
15563
15578
|
}
|
|
15564
15579
|
function simhash(text) {
|
|
15565
|
-
return simhashFeatures(shingle(
|
|
15580
|
+
return simhashFeatures(shingle(tokenize2(text)));
|
|
15566
15581
|
}
|
|
15567
15582
|
function hammingDistance(a, b) {
|
|
15568
15583
|
return popcount((a ^ b) & MASK64);
|
|
@@ -24085,7 +24100,7 @@ __export(src_exports, {
|
|
|
24085
24100
|
simhashFeatures: () => simhashFeatures,
|
|
24086
24101
|
symbolNodeId: () => symbolNodeId,
|
|
24087
24102
|
toHex: () => toHex,
|
|
24088
|
-
tokenize: () =>
|
|
24103
|
+
tokenize: () => tokenize2
|
|
24089
24104
|
});
|
|
24090
24105
|
function defaultProviders() {
|
|
24091
24106
|
if (providersCache) return providersCache;
|
|
@@ -25342,7 +25357,7 @@ function mergeDuplicateProblems(store2, opts) {
|
|
|
25342
25357
|
const report = { clusters: 0, merged: 0 };
|
|
25343
25358
|
const open = store2.findNodesByLabel("Problem").filter((p) => p.attrs["resolvedAt"] == null && p.attrs["mergedInto"] === void 0);
|
|
25344
25359
|
const tokens = /* @__PURE__ */ new Map();
|
|
25345
|
-
for (const p of open) tokens.set(p.id, new Set(
|
|
25360
|
+
for (const p of open) tokens.set(p.id, new Set(retrievalTokens(p.description)));
|
|
25346
25361
|
const consumed = /* @__PURE__ */ new Set();
|
|
25347
25362
|
store2.transaction(() => {
|
|
25348
25363
|
for (let i2 = 0; i2 < open.length; i2++) {
|