@namzu/sdk 53.2.0 → 53.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/compaction/salience/bm25.d.ts +1 -1
- package/dist/compaction/salience/bm25.d.ts.map +1 -1
- package/dist/compaction/salience/bm25.js +1 -2
- package/dist/compaction/salience/bm25.js.map +1 -1
- package/dist/compaction/salience/score.d.ts.map +1 -1
- package/dist/compaction/salience/score.js +1 -1
- package/dist/compaction/salience/score.js.map +1 -1
- package/dist/compaction/salience/tokenize.d.ts.map +1 -1
- package/dist/compaction/salience/tokenize.js +16 -2
- package/dist/compaction/salience/tokenize.js.map +1 -1
- package/package.json +1 -1
- package/src/compaction/salience/bm25.ts +4 -2
- package/src/compaction/salience/score.ts +3 -1
- package/src/compaction/salience/tokenize.ts +16 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 53.2.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 36e89ef: A turn whose tool inputs or results contain a very long unbroken run of capital letters, or a long run of dots, dashes or slashes inside one word, no longer freezes before the next model request. The compaction scorer split such a token with patterns (the capitals split and the separator trim) that took time quadratic in the length (about 30 seconds for 150,000 characters, per call) and tokenized every message twice; it now splits in linear time and tokenizes each message once. Scores, budgets and spills are unchanged; nothing to do on upgrade.
|
|
8
|
+
|
|
3
9
|
## 53.2.0
|
|
4
10
|
|
|
5
11
|
### Minor Changes
|
|
@@ -19,7 +19,7 @@ export interface Bm25Index {
|
|
|
19
19
|
readonly documentFrequency: ReadonlyMap<string, number>;
|
|
20
20
|
readonly averageLength: number;
|
|
21
21
|
}
|
|
22
|
-
export declare function indexDocument(text: string): Bm25Document;
|
|
22
|
+
export declare function indexDocument(text: string, tokens?: readonly string[]): Bm25Document;
|
|
23
23
|
export declare function buildIndex(documents: readonly Bm25Document[]): Bm25Index;
|
|
24
24
|
/** BM25 score of one document for a query given as tokens. */
|
|
25
25
|
export declare function bm25Score(index: Bm25Index, doc: Bm25Document, queryTokens: readonly string[]): number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../../src/compaction/salience/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,MAAM,WAAW,YAAY;IAC5B,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,YAAY,EAAE,CAAA;IAC3C,QAAQ,CAAC,iBAAiB,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACvD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC9B;AAKD,wBAAgB,aAAa,
|
|
1
|
+
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../../src/compaction/salience/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,MAAM,WAAW,YAAY;IAC5B,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,YAAY,EAAE,CAAA;IAC3C,QAAQ,CAAC,iBAAiB,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACvD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC9B;AAKD,wBAAgB,aAAa,CAC5B,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,SAAS,MAAM,EAAmB,GACxC,YAAY,CAEd;AAED,wBAAgB,UAAU,CAAC,SAAS,EAAE,SAAS,YAAY,EAAE,GAAG,SAAS,CAcxE;AAED,8DAA8D;AAC9D,wBAAgB,SAAS,CACxB,KAAK,EAAE,SAAS,EAChB,GAAG,EAAE,YAAY,EACjB,WAAW,EAAE,SAAS,MAAM,EAAE,GAC5B,MAAM,CAgBR"}
|
|
@@ -13,8 +13,7 @@
|
|
|
13
13
|
import { termFrequencies, tokenize } from './tokenize.js';
|
|
14
14
|
const K1 = 1.2;
|
|
15
15
|
const B = 0.75;
|
|
16
|
-
export function indexDocument(text) {
|
|
17
|
-
const tokens = tokenize(text);
|
|
16
|
+
export function indexDocument(text, tokens = tokenize(text)) {
|
|
18
17
|
return { tf: termFrequencies(tokens), length: tokens.length };
|
|
19
18
|
}
|
|
20
19
|
export function buildIndex(documents) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../../src/compaction/salience/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAazD,MAAM,EAAE,GAAG,GAAG,CAAA;AACd,MAAM,CAAC,GAAG,IAAI,CAAA;AAEd,MAAM,UAAU,aAAa,
|
|
1
|
+
{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../../src/compaction/salience/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAazD,MAAM,EAAE,GAAG,GAAG,CAAA;AACd,MAAM,CAAC,GAAG,IAAI,CAAA;AAEd,MAAM,UAAU,aAAa,CAC5B,IAAY,EACZ,SAA4B,QAAQ,CAAC,IAAI,CAAC;IAE1C,OAAO,EAAE,EAAE,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAkC;IAC5D,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC7B,KAAK,IAAI,GAAG,CAAC,MAAM,CAAA;QACnB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IACD,OAAO;QACN,SAAS;QACT,iBAAiB;QACjB,aAAa,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,MAAM;KACpE,CAAA;AACF,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,SAAS,CACxB,KAAgB,EAChB,GAAiB,EACjB,WAA8B;IAE9B,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IAC9D,MAAM,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAA;IAChC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,MAAM,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,CAAC,EAAE;YAAE,SAAQ;QACjB,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA;QACrD,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAA;QACpF,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../../src/compaction/salience/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAoB,OAAO,EAAe,MAAM,8BAA8B,CAAA;AAO1F,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC3B;AAED,eAAO,MAAM,wBAAwB,EAAE,eAKtC,CAAA;AAED,MAAM,WAAW,cAAc;IAC9B,+CAA+C;IAC/C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,2EAA2E;IAC3E,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAA;CACjC;AAED,eAAO,MAAM,uBAAuB,EAAE,cAIrC,CAAA;AAED,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAA;AAE3E,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC9B,+EAA+E;IAC/E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,yFAAyF;IACzF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAC1C;AAED,MAAM,WAAW,YAAY;IAC5B,6FAA6F;IAC7F,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,0DAA0D;IAC1D,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;CACzC;AAED,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAcpD;AAuBD,wBAAgB,aAAa,CAC5B,QAAQ,EAAE,SAAS,OAAO,EAAE,EAC5B,OAAO,EAAE,YAAY,GACnB,aAAa,EAAE,
|
|
1
|
+
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../../src/compaction/salience/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAoB,OAAO,EAAe,MAAM,8BAA8B,CAAA;AAO1F,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC3B;AAED,eAAO,MAAM,wBAAwB,EAAE,eAKtC,CAAA;AAED,MAAM,WAAW,cAAc;IAC9B,+CAA+C;IAC/C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,2EAA2E;IAC3E,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAA;CACjC;AAED,eAAO,MAAM,uBAAuB,EAAE,cAIrC,CAAA;AAED,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAA;AAE3E,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC9B,+EAA+E;IAC/E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,yFAAyF;IACzF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAC1C;AAED,MAAM,WAAW,YAAY;IAC5B,6FAA6F;IAC7F,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,0DAA0D;IAC1D,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;CACzC;AAED,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAcpD;AAuBD,wBAAgB,aAAa,CAC5B,QAAQ,EAAE,SAAS,OAAO,EAAE,EAC5B,OAAO,EAAE,YAAY,GACnB,aAAa,EAAE,CAsLjB"}
|
|
@@ -80,7 +80,7 @@ export function scoreMessages(messages, options) {
|
|
|
80
80
|
const texts = messages.map(messageText);
|
|
81
81
|
const richContent = messages.map(hasRichContent);
|
|
82
82
|
const tokenLists = texts.map(tokenize);
|
|
83
|
-
const documents = texts.map(indexDocument);
|
|
83
|
+
const documents = texts.map((text, i) => indexDocument(text, tokenLists[i]));
|
|
84
84
|
const index = buildIndex(documents);
|
|
85
85
|
const goalTokens = tokenize(options.goal);
|
|
86
86
|
// Relevance, normalised to the best message of the pass so weights mean
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"score.js","sourceRoot":"","sources":["../../../src/compaction/salience/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAA;AAC5D,OAAO,EAAqB,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACnF,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AASxC,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACxD,OAAO,EAAE,CAAC;IACV,SAAS,EAAE,CAAC;IACZ,OAAO,EAAE,CAAC;IACV,UAAU,EAAE,CAAC;CACb,CAAA;AAUD,MAAM,CAAC,MAAM,uBAAuB,GAAmB;IACtD,gBAAgB,EAAE,EAAE;IACpB,kBAAkB,EAAE,GAAG;IACvB,OAAO,EAAE,wBAAwB;CACjC,CAAA;AA0BD,4EAA4E;AAC5E,MAAM,UAAU,WAAW,CAAC,OAAgB;IAC3C,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC7B,OAAO,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YACzC,CAAC,CAAC,OAAO,CAAC,OAAO;YACjB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9F,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IACpE,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAClC,KAAK,MAAM,IAAI,IAAK,OAA4B,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;YAClE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;QACxD,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED,SAAS,cAAc,CAAC,OAAgB;IACvC,OAAO,CACN,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YACnC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CACxD,CAAA;AACF,CAAC;AAED,MAAM,oBAAoB,GAAG,CAAC,CAAU,EAAyB,EAAE,CAClE,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,OAAO,CAAE,CAAsB,CAAC,SAAS,CAAC,CAAA;AAE3E,gGAAgG;AAChG,SAAS,OAAO,CAAC,MAAyB;IACzC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACpF,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED,MAAM,UAAU,aAAa,CAC5B,QAA4B,EAC5B,OAAqB;IAErB,MAAM,MAAM,GAAmB;QAC9B,GAAG,uBAAuB;QAC1B,GAAG,OAAO,CAAC,MAAM;QACjB,OAAO,EAAE,EAAE,GAAG,wBAAwB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE;KACpE,CAAA;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAA;IACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IACvC,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IACtC,MAAM,SAAS,GAAmB,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"score.js","sourceRoot":"","sources":["../../../src/compaction/salience/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAA;AAC5D,OAAO,EAAqB,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACnF,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AASxC,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACxD,OAAO,EAAE,CAAC;IACV,SAAS,EAAE,CAAC;IACZ,OAAO,EAAE,CAAC;IACV,UAAU,EAAE,CAAC;CACb,CAAA;AAUD,MAAM,CAAC,MAAM,uBAAuB,GAAmB;IACtD,gBAAgB,EAAE,EAAE;IACpB,kBAAkB,EAAE,GAAG;IACvB,OAAO,EAAE,wBAAwB;CACjC,CAAA;AA0BD,4EAA4E;AAC5E,MAAM,UAAU,WAAW,CAAC,OAAgB;IAC3C,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC7B,OAAO,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YACzC,CAAC,CAAC,OAAO,CAAC,OAAO;YACjB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9F,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IACpE,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAClC,KAAK,MAAM,IAAI,IAAK,OAA4B,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;YAClE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;QACxD,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED,SAAS,cAAc,CAAC,OAAgB;IACvC,OAAO,CACN,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YACnC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CACxD,CAAA;AACF,CAAC;AAED,MAAM,oBAAoB,GAAG,CAAC,CAAU,EAAyB,EAAE,CAClE,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,OAAO,CAAE,CAAsB,CAAC,SAAS,CAAC,CAAA;AAE3E,gGAAgG;AAChG,SAAS,OAAO,CAAC,MAAyB;IACzC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACpF,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED,MAAM,UAAU,aAAa,CAC5B,QAA4B,EAC5B,OAAqB;IAErB,MAAM,MAAM,GAAmB;QAC9B,GAAG,uBAAuB;QAC1B,GAAG,OAAO,CAAC,MAAM;QACjB,OAAO,EAAE,EAAE,GAAG,wBAAwB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE;KACpE,CAAA;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAA;IACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IACvC,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IACtC,MAAM,SAAS,GAAmB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACvD,aAAa,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAa,CAAC,CAC9C,CAAA;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,CAAA;IACnC,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAEzC,wEAAwE;IACxE,gDAAgD;IAChD,uEAAuE;IACvE,uEAAuE;IACvE,iEAAiE;IACjE,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,CAAA;IAC9E,uEAAuE;IACvE,iEAAiE;IACjE,2DAA2D;IAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC5B,CAAC,EACD,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC/B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;QAChC,OAAO,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAA;IACjF,CAAC,CAAC,CACF,CAAA;IAED,yEAAyE;IACzE,wEAAwE;IACxE,sCAAsC;IACtC,MAAM,UAAU,GAAkB,IAAI,KAAK,CAAC,CAAC,CAAC,CAAA;IAC9C,IAAI,OAAO,GAAG,IAAI,GAAG,EAAU,CAAA;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,UAAU,CAAC,CAAC,CAAC,GAAG,OAAO,CAAA;QACvB,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;QACtC,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7D,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAa,CAAC,CAAC,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IAED,sEAAsE;IACtE,qEAAqE;IACrE,+BAA+B;IAC/B,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnC,oBAAoB,CAAC,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC9E,CAAC,CAAC,EAAE,CACL,CAAA;IACD,MAAM,aAAa,GAAkB,IAAI,KAAK,CAAC,CAAC,CAAC,CAAA;IACjD,IAAI,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,aAAa,CAAC,CAAC,CAAC,GAAG,KAAK,CAAA;QACxB,IAAK,QAAQ,CAAC,CAAC,CAAc,CAAC,MAAM,GAAG,CAAC;YACvC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,GAAI,QAAQ,CAAC,CAAC,CAAc,CAAC,CAAC,CAAA;IAC3D,CAAC;IAED,cAAc;IACd,MAAM,WAAW,GAA+B,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACvE,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,OAAO,QAAQ,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9D,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,CAAA;QACtC,QAAQ,IAAI,CAAC,CAAA;IACd,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,mBAAmB,CAAC,QAAQ,CAAC;QAC5C,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,WAAW,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAA;IACvD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAChF,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,WAAW,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAA;IACvD,CAAC;IACD,sDAAsD;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;QACtC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,SAAQ;QACvE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;YACxC,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC/B,IAAI,SAAS,CAAC,IAAI,KAAK,WAAW;oBAAE,MAAK;gBACzC,SAAQ;YACT,CAAC;YACD,IAAI,GAAG,CAAC,GAAG,CAAE,SAAyB,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpD,GAAG,CAAC,MAAM,CAAE,SAAyB,CAAC,UAAU,CAAC,CAAA;gBACjD,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI;oBAAE,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,CAAA;YACrD,CAAC;QACF,CAAC;IACF,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;QACtC,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,SAAQ;QAChE,MAAM,EAAE,GAAI,OAAuB,CAAC,UAAU,CAAA;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;YACxC,IAAI,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAC7F,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI;oBAAE,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,CAAA;gBACpD,MAAK;YACN,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAoB,EAAE,CAAA;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;QACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC1B,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAA;QACvE,MAAM,SAAS,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAE,YAAY,CAAC,CAAC,CAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAA;QAEnF,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,mEAAmE;YACnE,oEAAoE;YACpE,gDAAgD;YAChD,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAa,CAAA;YACxC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QACnE,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACpC,iEAAiE;YACjE,6DAA6D;YAC7D,gEAAgE;YAChE,iEAAiE;YACjE,gEAAgE;YAChE,eAAe;YACf,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAa,CAAC,CAAA;YAC/C,IAAI,IAAI,GAAG,CAAC,CAAA;YACZ,KAAK,MAAM,KAAK,IAAI,IAAI;gBAAE,IAAK,UAAU,CAAC,CAAC,CAAiB,CAAC,GAAG,CAAC,KAAK,CAAC;oBAAE,IAAI,IAAI,CAAC,CAAA;YAClF,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC3B,CAAC;aAAM,IAAI,oBAAoB,CAAC,OAAO,CAAC;YAAE,OAAO,GAAG,GAAG,CAAA;QAEvD,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAgB,CAAA;QAC9C,oEAAoE;QACpE,iEAAiE;QACjE,0EAA0E;QAC1E,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,IAAI,WAAW,CAAC,CAAC,CAAC,IAAK,QAAQ,CAAC,CAAC,CAAa,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI;oBAAE,SAAQ;gBAC9E,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAgB,CAAC,CAAA;gBAC/D,IAAI,GAAG,GAAG,UAAU;oBAAE,UAAU,GAAG,GAAG,CAAA;YACvC,CAAC;QACF,CAAC;QACD,IAAI,UAAU,GAAG,MAAM,CAAC,kBAAkB;YAAE,UAAU,GAAG,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;;YACtF,UAAU,GAAG,CAAC,CAAA;QACnB,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,+DAA+D;YAC/D,iCAAiC;YACjC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAY,CAAA;gBACvC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC;oBAAE,SAAQ;gBAC7C,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAM,OAAuB,CAAC,UAAU,CACnD,CAAA;gBACD,IAAI,IAAI,EAAE,CAAC;oBACV,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;oBAC9D,IAAK,aAAa,CAAC,CAAC,CAAiB,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,UAAU,GAAG,CAAC,CAAA;oBAC9D,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAA;QACxB,MAAM,QAAQ,GACb,CAAC,CAAC,OAAO,GAAG,OAAO;YACnB,CAAC,CAAC,SAAS,GAAG,SAAS;YACvB,CAAC,CAAC,OAAO,GAAG,OAAO;YACnB,CAAC,CAAC,UAAU,GAAG,UAAU,CAAA;QAC1B,MAAM,CAAC,IAAI,CAAC;YACX,KAAK,EAAE,CAAC;YACR,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC;YACtC,OAAO;YACP,SAAS;YACT,OAAO;YACP,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,WAAW,CAAC,CAAC,CAA2B;SACnD,CAAC,CAAA;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../../src/compaction/salience/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;
|
|
1
|
+
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../../src/compaction/salience/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAyDH,2EAA2E;AAC3E,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAuB/C;AAED,oCAAoC;AACpC,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAI9E"}
|
|
@@ -43,10 +43,24 @@ const STOP_WORDS = new Set([
|
|
|
43
43
|
'you',
|
|
44
44
|
]);
|
|
45
45
|
const WORD = /[\p{L}\p{N}_$][\p{L}\p{N}_$.\-/\\:]*/gu;
|
|
46
|
+
const SEPARATORS = new Set(['.', '-', '/', '\\', ':']);
|
|
47
|
+
// A linear scan from each end: a `[sep]+$` regex is retried from every start of a
|
|
48
|
+
// long separator run inside a token and fails at the end each time, quadratically.
|
|
49
|
+
function trimSeparators(token) {
|
|
50
|
+
let start = 0;
|
|
51
|
+
let end = token.length;
|
|
52
|
+
while (start < end && SEPARATORS.has(token[start]))
|
|
53
|
+
start++;
|
|
54
|
+
while (end > start && SEPARATORS.has(token[end - 1]))
|
|
55
|
+
end--;
|
|
56
|
+
return start === 0 && end === token.length ? token : token.slice(start, end);
|
|
57
|
+
}
|
|
46
58
|
function splitIdentifier(raw) {
|
|
47
59
|
const parts = raw
|
|
48
60
|
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
49
|
-
|
|
61
|
+
// Lookaround keeps this linear: a capturing `([A-Z]+)([A-Z][a-z])` backtracks
|
|
62
|
+
// quadratically on a long run of capitals that never reaches a lower-case letter.
|
|
63
|
+
.replace(/(?<=[A-Z])(?=[A-Z][a-z])/g, ' ')
|
|
50
64
|
.split(/[_\s]+/)
|
|
51
65
|
.filter((p) => p.length > 0);
|
|
52
66
|
return parts.length > 1 ? parts : [];
|
|
@@ -55,7 +69,7 @@ function splitIdentifier(raw) {
|
|
|
55
69
|
export function tokenize(text) {
|
|
56
70
|
const out = [];
|
|
57
71
|
for (const match of text.matchAll(WORD)) {
|
|
58
|
-
const raw = match[0]
|
|
72
|
+
const raw = trimSeparators(match[0]);
|
|
59
73
|
if (raw.length === 0)
|
|
60
74
|
continue;
|
|
61
75
|
const whole = raw.toLowerCase();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../../src/compaction/salience/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,UAAU,GAAwB,IAAI,GAAG,CAAC;IAC/C,GAAG;IACH,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,IAAI;IACJ,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;CACL,CAAC,CAAA;AAEF,MAAM,IAAI,GAAG,wCAAwC,CAAA;AAErD,SAAS,eAAe,CAAC,GAAW;IACnC,MAAM,KAAK,GAAG,GAAG;SACf,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../../src/compaction/salience/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,UAAU,GAAwB,IAAI,GAAG,CAAC;IAC/C,GAAG;IACH,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,IAAI;IACJ,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;CACL,CAAC,CAAA;AAEF,MAAM,IAAI,GAAG,wCAAwC,CAAA;AAErD,MAAM,UAAU,GAAwB,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAA;AAE3E,kFAAkF;AAClF,mFAAmF;AACnF,SAAS,cAAc,CAAC,KAAa;IACpC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAA;IACtB,OAAO,KAAK,GAAG,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAW,CAAC;QAAE,KAAK,EAAE,CAAA;IACrE,OAAO,GAAG,GAAG,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAW,CAAC;QAAE,GAAG,EAAE,CAAA;IACrE,OAAO,KAAK,KAAK,CAAC,IAAI,GAAG,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;AAC7E,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IACnC,MAAM,KAAK,GAAG,GAAG;SACf,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC;QACvC,8EAA8E;QAC9E,kFAAkF;SACjF,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;SACzC,KAAK,CAAC,QAAQ,CAAC;SACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;AACrC,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,QAAQ,CAAC,IAAY;IACpC,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QACpC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAA;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC3C,mEAAmE;QACnE,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAA;gBACnC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,KAAK;oBAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC9D,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,OAAO,CAAC;oBAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;aAAM,CAAC;YACP,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;gBAChC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,eAAe,CAAC,MAAyB;IACxD,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAA;IACpC,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACnE,OAAO,EAAE,CAAA;AACV,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@namzu/sdk",
|
|
3
|
-
"version": "53.2.
|
|
3
|
+
"version": "53.2.1",
|
|
4
4
|
"description": "Open-source AI kernel for TypeScript: runs an agent the way an operating system runs a process, with an identity and a budget, scheduled tool calls and checkpointed turns.",
|
|
5
5
|
"license": "FSL-1.1-MIT",
|
|
6
6
|
"engines": {
|
|
@@ -27,8 +27,10 @@ export interface Bm25Index {
|
|
|
27
27
|
const K1 = 1.2
|
|
28
28
|
const B = 0.75
|
|
29
29
|
|
|
30
|
-
export function indexDocument(
|
|
31
|
-
|
|
30
|
+
export function indexDocument(
|
|
31
|
+
text: string,
|
|
32
|
+
tokens: readonly string[] = tokenize(text),
|
|
33
|
+
): Bm25Document {
|
|
32
34
|
return { tf: termFrequencies(tokens), length: tokens.length }
|
|
33
35
|
}
|
|
34
36
|
|
|
@@ -132,7 +132,9 @@ export function scoreMessages(
|
|
|
132
132
|
const texts = messages.map(messageText)
|
|
133
133
|
const richContent = messages.map(hasRichContent)
|
|
134
134
|
const tokenLists = texts.map(tokenize)
|
|
135
|
-
const documents: Bm25Document[] = texts.map(
|
|
135
|
+
const documents: Bm25Document[] = texts.map((text, i) =>
|
|
136
|
+
indexDocument(text, tokenLists[i] as string[]),
|
|
137
|
+
)
|
|
136
138
|
const index = buildIndex(documents)
|
|
137
139
|
const goalTokens = tokenize(options.goal)
|
|
138
140
|
|
|
@@ -46,10 +46,24 @@ const STOP_WORDS: ReadonlySet<string> = new Set([
|
|
|
46
46
|
|
|
47
47
|
const WORD = /[\p{L}\p{N}_$][\p{L}\p{N}_$.\-/\\:]*/gu
|
|
48
48
|
|
|
49
|
+
const SEPARATORS: ReadonlySet<string> = new Set(['.', '-', '/', '\\', ':'])
|
|
50
|
+
|
|
51
|
+
// A linear scan from each end: a `[sep]+$` regex is retried from every start of a
|
|
52
|
+
// long separator run inside a token and fails at the end each time, quadratically.
|
|
53
|
+
function trimSeparators(token: string): string {
|
|
54
|
+
let start = 0
|
|
55
|
+
let end = token.length
|
|
56
|
+
while (start < end && SEPARATORS.has(token[start] as string)) start++
|
|
57
|
+
while (end > start && SEPARATORS.has(token[end - 1] as string)) end--
|
|
58
|
+
return start === 0 && end === token.length ? token : token.slice(start, end)
|
|
59
|
+
}
|
|
60
|
+
|
|
49
61
|
function splitIdentifier(raw: string): string[] {
|
|
50
62
|
const parts = raw
|
|
51
63
|
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
52
|
-
|
|
64
|
+
// Lookaround keeps this linear: a capturing `([A-Z]+)([A-Z][a-z])` backtracks
|
|
65
|
+
// quadratically on a long run of capitals that never reaches a lower-case letter.
|
|
66
|
+
.replace(/(?<=[A-Z])(?=[A-Z][a-z])/g, ' ')
|
|
53
67
|
.split(/[_\s]+/)
|
|
54
68
|
.filter((p) => p.length > 0)
|
|
55
69
|
return parts.length > 1 ? parts : []
|
|
@@ -59,7 +73,7 @@ function splitIdentifier(raw: string): string[] {
|
|
|
59
73
|
export function tokenize(text: string): string[] {
|
|
60
74
|
const out: string[] = []
|
|
61
75
|
for (const match of text.matchAll(WORD)) {
|
|
62
|
-
const raw = match[0]
|
|
76
|
+
const raw = trimSeparators(match[0])
|
|
63
77
|
if (raw.length === 0) continue
|
|
64
78
|
const whole = raw.toLowerCase()
|
|
65
79
|
if (!STOP_WORDS.has(whole)) out.push(whole)
|