@memberjunction/ai-vectors 5.48.0 → 5.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -73,6 +73,18 @@ graph TD
|
|
|
73
73
|
|
|
74
74
|
Token-aware text splitting that respects natural language boundaries. All methods are static.
|
|
75
75
|
|
|
76
|
+
> **Looking for chunking *strategy*?** `TextChunker` is the low-level primitive — given a string and a
|
|
77
|
+
> budget, split it. It has no notion of documents, media, or meaning. Choosing *where* content should
|
|
78
|
+
> be cut (headings, topic shifts, audio chapters) belongs to `BaseSegmenter` in
|
|
79
|
+
> [`@memberjunction/ai-segmentation`](../../Segmentation/README.md), which sits one layer up and uses
|
|
80
|
+
> `TextChunker` to enforce the budget within a unit it identified. Ingestion pipelines should call a
|
|
81
|
+
> segmenter, not this class directly. See the
|
|
82
|
+
> [Content Segmentation Guide](../../../../guides/CONTENT_SEGMENTATION_GUIDE.md).
|
|
83
|
+
>
|
|
84
|
+
> **Offsets are provenance.** `StartOffset`/`EndOffset` are persisted as chunk provenance and used to
|
|
85
|
+
> resolve a search hit back to its source passage — always verify that
|
|
86
|
+
> `text.slice(StartOffset, EndOffset)` contains the chunk when changing this class.
|
|
87
|
+
|
|
76
88
|
### Strategies
|
|
77
89
|
|
|
78
90
|
| Strategy | Splits On | Best For |
|
|
@@ -55,6 +55,18 @@ export declare class TextChunker {
|
|
|
55
55
|
static EstimateTokenCount(text: string): number;
|
|
56
56
|
private static chunkBySentence;
|
|
57
57
|
private static chunkByParagraph;
|
|
58
|
+
/**
|
|
59
|
+
* Resolve each unit's true offsets with a single forward-moving cursor.
|
|
60
|
+
*
|
|
61
|
+
* Searching from the start of the document for every unit (the obvious
|
|
62
|
+
* implementation) returns the *first* occurrence of that text, so any repeated
|
|
63
|
+
* sentence — boilerplate, a recurring header, "Thank you." — makes later chunks
|
|
64
|
+
* report offsets pointing at the wrong part of the document. Because those
|
|
65
|
+
* offsets are persisted as chunk provenance, that silently corrupts the link
|
|
66
|
+
* from a search hit back to its source passage. The cursor also makes this a
|
|
67
|
+
* single O(n) pass instead of O(n²).
|
|
68
|
+
*/
|
|
69
|
+
private static locateUnits;
|
|
58
70
|
private static chunkByFixed;
|
|
59
71
|
/**
|
|
60
72
|
* Split text into sentences using common sentence-ending punctuation.
|
|
@@ -66,12 +78,17 @@ export declare class TextChunker {
|
|
|
66
78
|
* the token limit, with overlap between consecutive chunks.
|
|
67
79
|
*/
|
|
68
80
|
private static mergeUnitsIntoChunks;
|
|
81
|
+
/**
|
|
82
|
+
* Total estimated tokens across a set of units.
|
|
83
|
+
*/
|
|
84
|
+
private static sumTokens;
|
|
69
85
|
/**
|
|
70
86
|
* Get the trailing units that fit within the overlap token budget.
|
|
71
87
|
*/
|
|
72
88
|
private static getOverlapUnits;
|
|
73
89
|
/**
|
|
74
|
-
* Build a TextChunk from
|
|
90
|
+
* Build a TextChunk from positioned units. Offsets come from the units
|
|
91
|
+
* themselves, which were resolved by a forward scan in `locateUnits`.
|
|
75
92
|
*/
|
|
76
93
|
private static buildChunkFromUnits;
|
|
77
94
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextChunker.d.ts","sourceRoot":"","sources":["../../src/generic/TextChunker.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC;CACjD;
|
|
1
|
+
{"version":3,"file":"TextChunker.d.ts","sourceRoot":"","sources":["../../src/generic/TextChunker.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC;CACjD;AAaD;;GAEG;AACH,MAAM,WAAW,SAAS;IACtB,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,qBAAa,WAAW;IACpB;;OAEG;WACW,SAAS,CAAC,MAAM,EAAE,eAAe,GAAG,SAAS,EAAE;IAsB7D;;;OAGG;WACW,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAUtD,OAAO,CAAC,MAAM,CAAC,eAAe;IAK9B,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAK/B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,MAAM,CAAC,WAAW;IAa1B,OAAO,CAAC,MAAM,CAAC,YAAY;IA+C3B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,cAAc;IAQ7B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,oBAAoB;IA4CnC;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;IAIxB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAgB9B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,mBAAmB;CAWrC"}
|
|
@@ -53,16 +53,43 @@ export class TextChunker {
|
|
|
53
53
|
// ─────────────────────────────────────────────
|
|
54
54
|
static chunkBySentence(text, maxTokens, overlapTokens) {
|
|
55
55
|
const sentences = TextChunker.splitSentences(text);
|
|
56
|
-
return TextChunker.mergeUnitsIntoChunks(sentences, text, maxTokens, overlapTokens);
|
|
56
|
+
return TextChunker.mergeUnitsIntoChunks(TextChunker.locateUnits(sentences, text), maxTokens, overlapTokens);
|
|
57
57
|
}
|
|
58
58
|
static chunkByParagraph(text, maxTokens, overlapTokens) {
|
|
59
59
|
const paragraphs = text.split(/\n\n+/).filter((p) => p.trim().length > 0);
|
|
60
|
-
return TextChunker.mergeUnitsIntoChunks(paragraphs, text, maxTokens, overlapTokens);
|
|
60
|
+
return TextChunker.mergeUnitsIntoChunks(TextChunker.locateUnits(paragraphs, text), maxTokens, overlapTokens);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Resolve each unit's true offsets with a single forward-moving cursor.
|
|
64
|
+
*
|
|
65
|
+
* Searching from the start of the document for every unit (the obvious
|
|
66
|
+
* implementation) returns the *first* occurrence of that text, so any repeated
|
|
67
|
+
* sentence — boilerplate, a recurring header, "Thank you." — makes later chunks
|
|
68
|
+
* report offsets pointing at the wrong part of the document. Because those
|
|
69
|
+
* offsets are persisted as chunk provenance, that silently corrupts the link
|
|
70
|
+
* from a search hit back to its source passage. The cursor also makes this a
|
|
71
|
+
* single O(n) pass instead of O(n²).
|
|
72
|
+
*/
|
|
73
|
+
static locateUnits(units, originalText) {
|
|
74
|
+
const positioned = [];
|
|
75
|
+
let cursor = 0;
|
|
76
|
+
for (const unit of units) {
|
|
77
|
+
const found = originalText.indexOf(unit, cursor);
|
|
78
|
+
const start = found >= 0 ? found : cursor;
|
|
79
|
+
const end = Math.min(start + unit.length, originalText.length);
|
|
80
|
+
positioned.push({ Text: unit, Start: start, End: end });
|
|
81
|
+
cursor = end;
|
|
82
|
+
}
|
|
83
|
+
return positioned;
|
|
61
84
|
}
|
|
62
85
|
static chunkByFixed(text, maxTokens, overlapTokens) {
|
|
63
|
-
const words = text.split(/\s+/);
|
|
64
86
|
const maxChars = maxTokens * 4; // rough token-to-char estimate
|
|
65
|
-
|
|
87
|
+
// Cap the overlap at half the window. Beyond 50% each chunk is mostly a copy
|
|
88
|
+
// of its predecessor, and as the overlap approaches the window size the chunk
|
|
89
|
+
// count explodes (an overlap >= the window never advances at all). The loop
|
|
90
|
+
// below additionally guarantees forward progress, since backing up to a word
|
|
91
|
+
// boundary can shorten a window enough that even a legal overlap would stall.
|
|
92
|
+
const overlapChars = Math.min(overlapTokens * 4, Math.floor(maxChars / 2));
|
|
66
93
|
const chunks = [];
|
|
67
94
|
let startCharOffset = 0;
|
|
68
95
|
let chunkIndex = 0;
|
|
@@ -85,11 +112,11 @@ export class TextChunker {
|
|
|
85
112
|
Index: chunkIndex++,
|
|
86
113
|
});
|
|
87
114
|
}
|
|
88
|
-
startCharOffset = endCharOffset - overlapChars;
|
|
89
|
-
if (startCharOffset >= text.length)
|
|
90
|
-
break;
|
|
91
115
|
if (endCharOffset >= text.length)
|
|
92
116
|
break;
|
|
117
|
+
// Always advance by at least one character, so a short word-boundary
|
|
118
|
+
// window can never leave the cursor where it started.
|
|
119
|
+
startCharOffset = Math.max(endCharOffset - overlapChars, startCharOffset + 1);
|
|
93
120
|
}
|
|
94
121
|
return chunks;
|
|
95
122
|
}
|
|
@@ -112,39 +139,45 @@ export class TextChunker {
|
|
|
112
139
|
* Merge small text units (sentences or paragraphs) into chunks that fit within
|
|
113
140
|
* the token limit, with overlap between consecutive chunks.
|
|
114
141
|
*/
|
|
115
|
-
static mergeUnitsIntoChunks(units,
|
|
142
|
+
static mergeUnitsIntoChunks(units, maxTokens, overlapTokens) {
|
|
116
143
|
const chunks = [];
|
|
117
144
|
let currentUnits = [];
|
|
118
145
|
let currentTokens = 0;
|
|
119
146
|
let chunkIndex = 0;
|
|
120
147
|
for (const unit of units) {
|
|
121
|
-
const unitTokens = TextChunker.EstimateTokenCount(unit);
|
|
148
|
+
const unitTokens = TextChunker.EstimateTokenCount(unit.Text);
|
|
122
149
|
// If a single unit exceeds the max, emit it as its own chunk
|
|
123
150
|
if (unitTokens > maxTokens) {
|
|
124
151
|
// Flush current buffer first
|
|
125
152
|
if (currentUnits.length > 0) {
|
|
126
|
-
chunks.push(TextChunker.buildChunkFromUnits(currentUnits,
|
|
153
|
+
chunks.push(TextChunker.buildChunkFromUnits(currentUnits, chunkIndex++));
|
|
127
154
|
currentUnits = TextChunker.getOverlapUnits(currentUnits, overlapTokens);
|
|
128
|
-
currentTokens =
|
|
155
|
+
currentTokens = TextChunker.sumTokens(currentUnits);
|
|
129
156
|
}
|
|
130
157
|
// Emit the oversized unit
|
|
131
|
-
chunks.push(TextChunker.buildChunkFromUnits([unit],
|
|
158
|
+
chunks.push(TextChunker.buildChunkFromUnits([unit], chunkIndex++));
|
|
132
159
|
continue;
|
|
133
160
|
}
|
|
134
161
|
if (currentTokens + unitTokens > maxTokens && currentUnits.length > 0) {
|
|
135
|
-
chunks.push(TextChunker.buildChunkFromUnits(currentUnits,
|
|
162
|
+
chunks.push(TextChunker.buildChunkFromUnits(currentUnits, chunkIndex++));
|
|
136
163
|
currentUnits = TextChunker.getOverlapUnits(currentUnits, overlapTokens);
|
|
137
|
-
currentTokens =
|
|
164
|
+
currentTokens = TextChunker.sumTokens(currentUnits);
|
|
138
165
|
}
|
|
139
166
|
currentUnits.push(unit);
|
|
140
167
|
currentTokens += unitTokens;
|
|
141
168
|
}
|
|
142
169
|
// Flush remaining
|
|
143
170
|
if (currentUnits.length > 0) {
|
|
144
|
-
chunks.push(TextChunker.buildChunkFromUnits(currentUnits,
|
|
171
|
+
chunks.push(TextChunker.buildChunkFromUnits(currentUnits, chunkIndex));
|
|
145
172
|
}
|
|
146
173
|
return chunks;
|
|
147
174
|
}
|
|
175
|
+
/**
|
|
176
|
+
* Total estimated tokens across a set of units.
|
|
177
|
+
*/
|
|
178
|
+
static sumTokens(units) {
|
|
179
|
+
return units.reduce((sum, u) => sum + TextChunker.EstimateTokenCount(u.Text), 0);
|
|
180
|
+
}
|
|
148
181
|
/**
|
|
149
182
|
* Get the trailing units that fit within the overlap token budget.
|
|
150
183
|
*/
|
|
@@ -154,7 +187,7 @@ export class TextChunker {
|
|
|
154
187
|
const overlapUnits = [];
|
|
155
188
|
let tokens = 0;
|
|
156
189
|
for (let i = units.length - 1; i >= 0; i--) {
|
|
157
|
-
const unitTokens = TextChunker.EstimateTokenCount(units[i]);
|
|
190
|
+
const unitTokens = TextChunker.EstimateTokenCount(units[i].Text);
|
|
158
191
|
if (tokens + unitTokens > overlapTokens)
|
|
159
192
|
break;
|
|
160
193
|
overlapUnits.unshift(units[i]);
|
|
@@ -163,17 +196,15 @@ export class TextChunker {
|
|
|
163
196
|
return overlapUnits;
|
|
164
197
|
}
|
|
165
198
|
/**
|
|
166
|
-
* Build a TextChunk from
|
|
199
|
+
* Build a TextChunk from positioned units. Offsets come from the units
|
|
200
|
+
* themselves, which were resolved by a forward scan in `locateUnits`.
|
|
167
201
|
*/
|
|
168
|
-
static buildChunkFromUnits(units,
|
|
169
|
-
const text = units.join(' ');
|
|
170
|
-
const startOffset = originalText.indexOf(units[0]);
|
|
171
|
-
const lastUnit = units[units.length - 1];
|
|
172
|
-
const endOffset = originalText.indexOf(lastUnit, startOffset) + lastUnit.length;
|
|
202
|
+
static buildChunkFromUnits(units, index) {
|
|
203
|
+
const text = units.map((u) => u.Text).join(' ');
|
|
173
204
|
return {
|
|
174
205
|
Text: text,
|
|
175
|
-
StartOffset:
|
|
176
|
-
EndOffset:
|
|
206
|
+
StartOffset: units[0].Start,
|
|
207
|
+
EndOffset: units[units.length - 1].End,
|
|
177
208
|
TokenCount: TextChunker.EstimateTokenCount(text),
|
|
178
209
|
Index: index,
|
|
179
210
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextChunker.js","sourceRoot":"","sources":["../../src/generic/TextChunker.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;
|
|
1
|
+
{"version":3,"file":"TextChunker.js","sourceRoot":"","sources":["../../src/generic/TextChunker.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AA2CH;;;;;;;;GAQG;AACH,MAAM,OAAO,WAAW;IACpB;;OAEG;IACI,MAAM,CAAC,SAAS,CAAC,MAAuB;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,IAAI,GAAG,CAAC;QAC/C,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;QAC1E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC;QAE/C,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,UAAU;gBACX,OAAO,WAAW,CAAC,eAAe,CAAC,IAAI,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACvE,KAAK,WAAW;gBACZ,OAAO,WAAW,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACxE,KAAK,OAAO;gBACR,OAAO,WAAW,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACpE;gBACI,OAAO,WAAW,CAAC,eAAe,CAAC,IAAI,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,kBAAkB,CAAC,IAAY;QACzC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAChD,gEAAgE;QAChE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,gDAAgD;IAChD,2BAA2B;IAC3B,gDAAgD;IAExC,MAAM,CAAC,eAAe,CAAC,IAAY,EAAE,SAAiB,EAAE,aAAqB;QACjF,MAAM,SAAS,GAAG,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACnD,OAAO,WAAW,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAChH,CAAC;IAEO,MAAM,CAAC,gBAAgB,CAAC,IAAY,EAAE,SAAiB,EAAE,aAAqB;QAClF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1E,OAAO,WAAW,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACjH,CAAC;IAED;;;;;;;;;;OAUG;IACK,MAAM,CAAC,WAAW,CAAC,KAAe,EAAE,YAAoB;QAC5D,MAAM,UAAU,GAAqB,EAAE,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;YAC/D,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACxD,MAAM,GAAG,GAAG,CAAC;QACjB,CAAC;QACD,OAAO,UAAU,CAAC;IACtB,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,IAAY,EAAE,SAAiB,EAAE,aAAqB;QAC9E,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,+BAA+B;QAC/D,6EAA6E;QAC7E,8EAA8E;QAC9E,4EAA4E;QAC5E,6EAA6E;QAC7E,8EAA8E;QAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,OAAO,eAAe,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAEtE,2CAA2C;YAC3C,IAAI,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;gBACvD,IAAI,SAAS,GAAG,eAAe,EAAE,CAAC;oBAC9B,aAAa,GAAG,SAAS,CAAC;gBAC9B,CAAC;YACL,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC;YACpE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,eAAe;oBAC5B,SAAS,EAAE,aAAa;oBACxB,UAAU,EAAE,WAAW,CAAC,kBAAkB,CAAC,SAAS,CAAC;oBACrD,KAAK,EAAE,UAAU,EAAE;iBACtB,CAAC,CAAC;YACP,CAAC;YAED,IAAI,aAAa,IAAI,IAAI,CAAC,MAAM;gBAAE,MAAM;YACxC,qEAAqE;YACrE,sDAAsD;YACtD,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,YAAY,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,gDAAgD;IAChD,kBAAkB;IAClB,gDAAgD;IAEhD;;;OAGG;IACK,MAAM,CAAC,cAAc,CAAC,IAAY;QACtC,0EAA0E;QAC1E,MAAM,aAAa,GAAG,iCAAiC,CAAC;QACxD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,oBAAoB,CAC/B,KAAuB,EACvB,SAAiB,EACjB,aAAqB;QAErB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,YAAY,GAAqB,EAAE,CAAC;QACxC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,UAAU,GAAG,WAAW,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7D,6DAA6D;YAC7D,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;gBACzB,6BAA6B;gBAC7B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;oBACzE,YAAY,GAAG,WAAW,CAAC,eAAe,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;oBACxE,aAAa,GAAG,WAAW,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;gBACxD,CAAC;gBACD,0BAA0B;gBAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;gBACnE,SAAS;YACb,CAAC;YAED,IAAI,aAAa,GAAG,UAAU,GAAG,SAAS,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;gBACzE,YAAY,GAAG,WAAW,CAAC,eAAe,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;gBACxE,aAAa,GAAG,WAAW,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACxD,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,aAAa,IAAI,UAAU,CAAC;QAChC,CAAC;QAED,kBAAkB;QAClB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;QAC3E,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,SAAS,CAAC,KAAuB;QAC5C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,eAAe,CAAC,KAAuB,EAAE,aAAqB;QACzE,IAAI,aAAa,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,YAAY,GAAqB,EAAE,CAAC;QAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,UAAU,GAAG,WAAW,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACjE,IAAI,MAAM,GAAG,UAAU,GAAG,aAAa;gBAAE,MAAM;YAC/C,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,IAAI,UAAU,CAAC;QACzB,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,mBAAmB,CAAC,KAAuB,EAAE,KAAa;QACrE,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO;YACH,IAAI,EAAE,IAAI;YACV,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;YAC3B,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG;YACtC,UAAU,EAAE,WAAW,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAChD,KAAK,EAAE,KAAK;SACf,CAAC;IACN,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/ai-vectors",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.50.0",
|
|
5
5
|
"description": "MemberJunction: AI Vectors Module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
@@ -17,13 +17,13 @@
|
|
|
17
17
|
"author": "MemberJunction.com",
|
|
18
18
|
"license": "ISC",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@memberjunction/core": "5.
|
|
21
|
-
"@memberjunction/global": "5.
|
|
22
|
-
"@memberjunction/core-entities": "5.
|
|
23
|
-
"@memberjunction/aiengine": "5.
|
|
24
|
-
"@memberjunction/ai": "5.
|
|
25
|
-
"@memberjunction/ai-core-plus": "5.
|
|
26
|
-
"@memberjunction/ai-vectordb": "5.
|
|
20
|
+
"@memberjunction/core": "5.50.0",
|
|
21
|
+
"@memberjunction/global": "5.50.0",
|
|
22
|
+
"@memberjunction/core-entities": "5.50.0",
|
|
23
|
+
"@memberjunction/aiengine": "5.50.0",
|
|
24
|
+
"@memberjunction/ai": "5.50.0",
|
|
25
|
+
"@memberjunction/ai-core-plus": "5.50.0",
|
|
26
|
+
"@memberjunction/ai-vectordb": "5.50.0",
|
|
27
27
|
"dotenv": "^17.2.4",
|
|
28
28
|
"openai": "^6.18.0"
|
|
29
29
|
},
|