@anuma/sdk 6.4.1 → 6.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/{chunk-WYARTNEN.mjs → chunk-NCP3VGRX.mjs} +232 -7
- package/dist/expo/index.cjs +268 -153
- package/dist/expo/index.d.mts +163 -7
- package/dist/expo/index.d.ts +163 -7
- package/dist/expo/index.mjs +12 -148
- package/dist/expo/{recall-7FOWWX5T.mjs → recall-ISSI7RMD.mjs} +1 -1
- package/dist/index.cjs +71 -0
- package/dist/index.d.mts +121 -1
- package/dist/index.d.ts +121 -1
- package/dist/index.mjs +67 -0
- package/dist/react/{chunk-IO4ZCPK2.mjs → chunk-OB5HFCLB.mjs} +253 -7
- package/dist/react/index.cjs +293 -203
- package/dist/react/index.d.mts +163 -7
- package/dist/react/index.d.ts +163 -7
- package/dist/react/index.mjs +15 -196
- package/dist/react/{recall-2J2755WC.mjs → recall-DQJINKEY.mjs} +1 -1
- package/dist/server/{chunk-EKYGWD5N.mjs → chunk-BZEAYCBV.mjs} +97 -7
- package/dist/server/index.cjs +120 -45
- package/dist/server/index.d.mts +54 -6
- package/dist/server/index.d.ts +54 -6
- package/dist/server/index.mjs +3 -37
- package/dist/server/{recall-IVV6EILK.mjs → recall-CZFWWOVF.mjs} +1 -1
- package/package.json +3 -3
package/dist/server/index.cjs
CHANGED
|
@@ -15911,6 +15911,90 @@ var init_vector = __esm({
|
|
|
15911
15911
|
}
|
|
15912
15912
|
});
|
|
15913
15913
|
|
|
15914
|
+
// src/lib/processors/encoding.ts
|
|
15915
|
+
async function dataUrlToArrayBuffer(dataUrl) {
|
|
15916
|
+
if (dataUrl.startsWith("blob:") || dataUrl.startsWith("http://") || dataUrl.startsWith("https://")) {
|
|
15917
|
+
const response = await fetch(dataUrl);
|
|
15918
|
+
return response.arrayBuffer();
|
|
15919
|
+
}
|
|
15920
|
+
const base64 = dataUrl.split(",")[1];
|
|
15921
|
+
if (typeof Buffer !== "undefined") {
|
|
15922
|
+
const buf = Buffer.from(base64, "base64");
|
|
15923
|
+
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
15924
|
+
}
|
|
15925
|
+
const binary = atob(base64);
|
|
15926
|
+
const buffer = new ArrayBuffer(binary.length);
|
|
15927
|
+
const view = new Uint8Array(buffer);
|
|
15928
|
+
for (let i = 0; i < binary.length; i++) {
|
|
15929
|
+
view[i] = binary.charCodeAt(i);
|
|
15930
|
+
}
|
|
15931
|
+
return buffer;
|
|
15932
|
+
}
|
|
15933
|
+
function uint8ArrayToBase64(data) {
|
|
15934
|
+
if (typeof Buffer !== "undefined") {
|
|
15935
|
+
return Buffer.from(data).toString("base64");
|
|
15936
|
+
}
|
|
15937
|
+
let base64 = "";
|
|
15938
|
+
for (let i = 0; i < data.length; i += B64_ENCODE_CHUNK) {
|
|
15939
|
+
const slice = data.subarray(i, i + B64_ENCODE_CHUNK);
|
|
15940
|
+
let binary = "";
|
|
15941
|
+
for (let j = 0; j < slice.length; j++) {
|
|
15942
|
+
binary += String.fromCharCode(slice[j]);
|
|
15943
|
+
}
|
|
15944
|
+
base64 += btoa(binary);
|
|
15945
|
+
}
|
|
15946
|
+
return base64;
|
|
15947
|
+
}
|
|
15948
|
+
function base64ToUint8Array(base64) {
|
|
15949
|
+
if (typeof Buffer !== "undefined") {
|
|
15950
|
+
const buf = Buffer.from(base64, "base64");
|
|
15951
|
+
return new Uint8Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
|
|
15952
|
+
}
|
|
15953
|
+
const binary = atob(base64);
|
|
15954
|
+
const bytes = new Uint8Array(binary.length);
|
|
15955
|
+
for (let i = 0; i < binary.length; i++) {
|
|
15956
|
+
bytes[i] = binary.charCodeAt(i);
|
|
15957
|
+
}
|
|
15958
|
+
return bytes;
|
|
15959
|
+
}
|
|
15960
|
+
var B64_ENCODE_CHUNK;
|
|
15961
|
+
var init_encoding = __esm({
|
|
15962
|
+
"src/lib/processors/encoding.ts"() {
|
|
15963
|
+
"use strict";
|
|
15964
|
+
B64_ENCODE_CHUNK = 32768 * 3;
|
|
15965
|
+
}
|
|
15966
|
+
});
|
|
15967
|
+
|
|
15968
|
+
// src/lib/memoryEngine/vectorEncoding.ts
|
|
15969
|
+
function decodeChunkVector(vector, onMalformed) {
|
|
15970
|
+
if (!vector || vector.length === 0) return new Float32Array(0);
|
|
15971
|
+
if (typeof vector !== "string") return Float32Array.from(vector);
|
|
15972
|
+
if (vector.length % 4 !== 0 || !CANONICAL_BASE64.test(vector)) {
|
|
15973
|
+
onMalformed?.();
|
|
15974
|
+
return new Float32Array(0);
|
|
15975
|
+
}
|
|
15976
|
+
try {
|
|
15977
|
+
const bytes = base64ToUint8Array(vector);
|
|
15978
|
+
if (bytes.byteLength === 0 || bytes.byteLength % BYTES_PER_FLOAT32 !== 0) {
|
|
15979
|
+
onMalformed?.();
|
|
15980
|
+
return new Float32Array(0);
|
|
15981
|
+
}
|
|
15982
|
+
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / BYTES_PER_FLOAT32);
|
|
15983
|
+
} catch {
|
|
15984
|
+
onMalformed?.();
|
|
15985
|
+
return new Float32Array(0);
|
|
15986
|
+
}
|
|
15987
|
+
}
|
|
15988
|
+
var BYTES_PER_FLOAT32, CANONICAL_BASE64;
|
|
15989
|
+
var init_vectorEncoding = __esm({
|
|
15990
|
+
"src/lib/memoryEngine/vectorEncoding.ts"() {
|
|
15991
|
+
"use strict";
|
|
15992
|
+
init_encoding();
|
|
15993
|
+
BYTES_PER_FLOAT32 = 4;
|
|
15994
|
+
CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
|
|
15995
|
+
}
|
|
15996
|
+
});
|
|
15997
|
+
|
|
15914
15998
|
// src/lib/db/chat/conversationEncryption.ts
|
|
15915
15999
|
async function encryptConversationFields(conversation, address, signMessage, embeddedWalletSigner) {
|
|
15916
16000
|
if (!address || !signMessage) {
|
|
@@ -16797,6 +16881,7 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16797
16881
|
const messages = await ctx.messagesCollection.query(...queryConditions).fetch();
|
|
16798
16882
|
const candidates = [];
|
|
16799
16883
|
let staleSkipped = 0;
|
|
16884
|
+
let unreadableVectors = 0;
|
|
16800
16885
|
for (const message of messages) {
|
|
16801
16886
|
const msgConvId = String(message._getRaw("conversation_id"));
|
|
16802
16887
|
if (!activeConversationIds.has(msgConvId)) continue;
|
|
@@ -16834,7 +16919,9 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16834
16919
|
const chunks = await readJsonField(message, "chunks", ctx.walletAddress);
|
|
16835
16920
|
if (chunks && chunks.length > 0) {
|
|
16836
16921
|
const vectors = chunks.map(
|
|
16837
|
-
(chunk) => chunk.vector
|
|
16922
|
+
(chunk) => decodeChunkVector(chunk.vector, () => {
|
|
16923
|
+
unreadableVectors++;
|
|
16924
|
+
})
|
|
16838
16925
|
);
|
|
16839
16926
|
chunkCache?.set(message.id, { version, chunks: vectors });
|
|
16840
16927
|
for (let ci = 0; ci < chunks.length; ci++) {
|
|
@@ -16865,6 +16952,11 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16865
16952
|
`searchChunksOp: skipped ${staleSkipped} messages whose embedding model differs from the current model (${embeddingModel}) \u2014 re-embed via chunkAndEmbedAllMessages`
|
|
16866
16953
|
);
|
|
16867
16954
|
}
|
|
16955
|
+
if (unreadableVectors > 0) {
|
|
16956
|
+
getLogger().warn(
|
|
16957
|
+
`searchChunksOp: could not read ${unreadableVectors} chunk vectors and left them out of ranking \u2014 the stored value is corrupt, so those chunks stay unsearchable until the message is re-embedded via chunkAndEmbedAllMessages`
|
|
16958
|
+
);
|
|
16959
|
+
}
|
|
16868
16960
|
const topK = candidates.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
|
|
16869
16961
|
const uniqueMessages = /* @__PURE__ */ new Map();
|
|
16870
16962
|
for (const c of topK) uniqueMessages.set(c.message.id, c.message);
|
|
@@ -16986,6 +17078,7 @@ var init_operations = __esm({
|
|
|
16986
17078
|
import_uuid2 = require("uuid");
|
|
16987
17079
|
init_logger();
|
|
16988
17080
|
init_vector();
|
|
17081
|
+
init_vectorEncoding();
|
|
16989
17082
|
init_encryption_utils();
|
|
16990
17083
|
init_conversationEncryption();
|
|
16991
17084
|
init_encryption();
|
|
@@ -18657,6 +18750,15 @@ var init_chunking = __esm({
|
|
|
18657
18750
|
}
|
|
18658
18751
|
});
|
|
18659
18752
|
|
|
18753
|
+
// src/lib/chat/toolResults.ts
|
|
18754
|
+
var TOOL_RESULT_ORIGIN;
|
|
18755
|
+
var init_toolResults = __esm({
|
|
18756
|
+
"src/lib/chat/toolResults.ts"() {
|
|
18757
|
+
"use strict";
|
|
18758
|
+
TOOL_RESULT_ORIGIN = "tool_result";
|
|
18759
|
+
}
|
|
18760
|
+
});
|
|
18761
|
+
|
|
18660
18762
|
// src/lib/memoryEngine/constants.ts
|
|
18661
18763
|
var DEFAULT_API_EMBEDDING_MODEL;
|
|
18662
18764
|
var init_constants = __esm({
|
|
@@ -19742,6 +19844,11 @@ var init_generate = __esm({
|
|
|
19742
19844
|
});
|
|
19743
19845
|
|
|
19744
19846
|
// src/lib/memoryEngine/embeddings.ts
|
|
19847
|
+
function isNonEmbeddableOrigin(origin, reembedDiscarded = false) {
|
|
19848
|
+
if (typeof origin !== "string") return false;
|
|
19849
|
+
if (reembedDiscarded && origin === CHUNKS_DISCARDED_ORIGIN) return false;
|
|
19850
|
+
return NON_EMBEDDABLE_ORIGINS.has(origin);
|
|
19851
|
+
}
|
|
19745
19852
|
async function embedMessage(ctx, messageId, options) {
|
|
19746
19853
|
const message = await getMessageOp(ctx, messageId);
|
|
19747
19854
|
if (!message) {
|
|
@@ -19750,7 +19857,7 @@ async function embedMessage(ctx, messageId, options) {
|
|
|
19750
19857
|
if (message.vector && message.vector.length > 0) {
|
|
19751
19858
|
return message;
|
|
19752
19859
|
}
|
|
19753
|
-
if (message.origin
|
|
19860
|
+
if (isNonEmbeddableOrigin(message.origin)) {
|
|
19754
19861
|
return message;
|
|
19755
19862
|
}
|
|
19756
19863
|
if (isEncrypted(message.content)) {
|
|
@@ -19786,7 +19893,7 @@ async function embedAllMessages(ctx, options, filter) {
|
|
|
19786
19893
|
if (message.role === "system") {
|
|
19787
19894
|
continue;
|
|
19788
19895
|
}
|
|
19789
|
-
if (message.origin
|
|
19896
|
+
if (isNonEmbeddableOrigin(message.origin, filter?.reembedDiscarded)) {
|
|
19790
19897
|
continue;
|
|
19791
19898
|
}
|
|
19792
19899
|
considered++;
|
|
@@ -19826,7 +19933,7 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
|
|
|
19826
19933
|
if (message.chunks && message.chunks.length > 0) {
|
|
19827
19934
|
return message;
|
|
19828
19935
|
}
|
|
19829
|
-
if (message.origin
|
|
19936
|
+
if (isNonEmbeddableOrigin(message.origin)) {
|
|
19830
19937
|
return message;
|
|
19831
19938
|
}
|
|
19832
19939
|
if (isEncrypted(message.content)) {
|
|
@@ -19874,7 +19981,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
|
|
|
19874
19981
|
if (hasVector && !filter?.rechunkExisting && !isStale) continue;
|
|
19875
19982
|
if (filter?.roles && !filter.roles.includes(message.role)) continue;
|
|
19876
19983
|
if (message.role === "system") continue;
|
|
19877
|
-
if (message.origin
|
|
19984
|
+
if (isNonEmbeddableOrigin(message.origin, filter?.reembedDiscarded)) continue;
|
|
19878
19985
|
considered++;
|
|
19879
19986
|
if (isEncrypted(message.content)) {
|
|
19880
19987
|
stillEncrypted++;
|
|
@@ -19943,10 +20050,11 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
|
|
|
19943
20050
|
}
|
|
19944
20051
|
return embeddedCount;
|
|
19945
20052
|
}
|
|
19946
|
-
var DEFAULT_MIN_CONTENT_LENGTH,
|
|
20053
|
+
var DEFAULT_MIN_CONTENT_LENGTH, CHUNKS_DISCARDED_ORIGIN, NON_EMBEDDABLE_ORIGINS;
|
|
19947
20054
|
var init_embeddings = __esm({
|
|
19948
20055
|
"src/lib/memoryEngine/embeddings.ts"() {
|
|
19949
20056
|
"use strict";
|
|
20057
|
+
init_toolResults();
|
|
19950
20058
|
init_operations();
|
|
19951
20059
|
init_encryption_utils();
|
|
19952
20060
|
init_logger();
|
|
@@ -19955,7 +20063,8 @@ var init_embeddings = __esm({
|
|
|
19955
20063
|
init_generate();
|
|
19956
20064
|
init_generate();
|
|
19957
20065
|
DEFAULT_MIN_CONTENT_LENGTH = 10;
|
|
19958
|
-
|
|
20066
|
+
CHUNKS_DISCARDED_ORIGIN = "chunks_discarded";
|
|
20067
|
+
NON_EMBEDDABLE_ORIGINS = /* @__PURE__ */ new Set([TOOL_RESULT_ORIGIN, CHUNKS_DISCARDED_ORIGIN]);
|
|
19959
20068
|
}
|
|
19960
20069
|
});
|
|
19961
20070
|
|
|
@@ -32619,44 +32728,7 @@ var WalletPoller = class {
|
|
|
32619
32728
|
|
|
32620
32729
|
// src/lib/processors/ExcelProcessor.ts
|
|
32621
32730
|
init_logger();
|
|
32622
|
-
|
|
32623
|
-
// src/lib/processors/encoding.ts
|
|
32624
|
-
var B64_ENCODE_CHUNK = 32768 * 3;
|
|
32625
|
-
async function dataUrlToArrayBuffer(dataUrl) {
|
|
32626
|
-
if (dataUrl.startsWith("blob:") || dataUrl.startsWith("http://") || dataUrl.startsWith("https://")) {
|
|
32627
|
-
const response = await fetch(dataUrl);
|
|
32628
|
-
return response.arrayBuffer();
|
|
32629
|
-
}
|
|
32630
|
-
const base64 = dataUrl.split(",")[1];
|
|
32631
|
-
if (typeof Buffer !== "undefined") {
|
|
32632
|
-
const buf = Buffer.from(base64, "base64");
|
|
32633
|
-
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
32634
|
-
}
|
|
32635
|
-
const binary = atob(base64);
|
|
32636
|
-
const buffer = new ArrayBuffer(binary.length);
|
|
32637
|
-
const view = new Uint8Array(buffer);
|
|
32638
|
-
for (let i = 0; i < binary.length; i++) {
|
|
32639
|
-
view[i] = binary.charCodeAt(i);
|
|
32640
|
-
}
|
|
32641
|
-
return buffer;
|
|
32642
|
-
}
|
|
32643
|
-
function uint8ArrayToBase64(data) {
|
|
32644
|
-
if (typeof Buffer !== "undefined") {
|
|
32645
|
-
return Buffer.from(data).toString("base64");
|
|
32646
|
-
}
|
|
32647
|
-
let base64 = "";
|
|
32648
|
-
for (let i = 0; i < data.length; i += B64_ENCODE_CHUNK) {
|
|
32649
|
-
const slice = data.subarray(i, i + B64_ENCODE_CHUNK);
|
|
32650
|
-
let binary = "";
|
|
32651
|
-
for (let j = 0; j < slice.length; j++) {
|
|
32652
|
-
binary += String.fromCharCode(slice[j]);
|
|
32653
|
-
}
|
|
32654
|
-
base64 += btoa(binary);
|
|
32655
|
-
}
|
|
32656
|
-
return base64;
|
|
32657
|
-
}
|
|
32658
|
-
|
|
32659
|
-
// src/lib/processors/ExcelProcessor.ts
|
|
32731
|
+
init_encoding();
|
|
32660
32732
|
if (typeof process !== "undefined" && typeof process.umask !== "function") {
|
|
32661
32733
|
process.umask = (_mask) => 18;
|
|
32662
32734
|
}
|
|
@@ -32835,6 +32907,7 @@ var PdfProcessor = class {
|
|
|
32835
32907
|
|
|
32836
32908
|
// src/lib/processors/TextProcessor.ts
|
|
32837
32909
|
init_logger();
|
|
32910
|
+
init_encoding();
|
|
32838
32911
|
var MIME = {
|
|
32839
32912
|
PLAIN: "text/plain",
|
|
32840
32913
|
MARKDOWN: "text/markdown",
|
|
@@ -32918,6 +32991,7 @@ var TextProcessor = class {
|
|
|
32918
32991
|
// src/lib/processors/WordProcessor.ts
|
|
32919
32992
|
var import_mammoth = __toESM(require("mammoth"));
|
|
32920
32993
|
init_logger();
|
|
32994
|
+
init_encoding();
|
|
32921
32995
|
var WordProcessor = class {
|
|
32922
32996
|
constructor() {
|
|
32923
32997
|
this.name = "word";
|
|
@@ -32961,6 +33035,7 @@ var WordProcessor = class {
|
|
|
32961
33035
|
// src/lib/processors/ZipProcessor.ts
|
|
32962
33036
|
var import_jszip = __toESM(require("jszip"));
|
|
32963
33037
|
init_logger();
|
|
33038
|
+
init_encoding();
|
|
32964
33039
|
var ZipProcessor = class {
|
|
32965
33040
|
constructor(options = {}) {
|
|
32966
33041
|
this.name = "zip";
|
package/dist/server/index.d.mts
CHANGED
|
@@ -3423,13 +3423,24 @@ type ChatRole = "user" | "assistant" | "system";
|
|
|
3423
3423
|
*/
|
|
3424
3424
|
type MessageFeedback = "like" | "dislike" | null;
|
|
3425
3425
|
/**
|
|
3426
|
-
*
|
|
3427
|
-
*
|
|
3428
|
-
*
|
|
3426
|
+
* Provenance for a row that needs handling the content cannot justify. Set at
|
|
3427
|
+
* write time by whoever produced or repaired the row; a union so further kinds
|
|
3428
|
+
* can be added without another column.
|
|
3429
|
+
*
|
|
3430
|
+
* Read it as an enum, never as a boolean. The values do NOT share a meaning:
|
|
3431
|
+
* `tool_result` marks a synthetic row that is both unindexed and hidden, while
|
|
3432
|
+
* `chunks_discarded` marks an ordinary typed message that is unindexed and still
|
|
3433
|
+
* rendered. Any predicate that collapses this to "has an origin" hides real
|
|
3434
|
+
* messages (see `isToolResultsRow`).
|
|
3429
3435
|
*
|
|
3430
3436
|
* - `tool_result`: the hidden `[Tool Execution Results]` row built from a
|
|
3431
3437
|
* turn's auto-executed tool results. Skipped by the embedding sweep (see
|
|
3432
|
-
* `memoryEngine/embeddings`).
|
|
3438
|
+
* `memoryEngine/embeddings`) AND hidden from the transcript.
|
|
3439
|
+
* - `chunks_discarded`: an ordinary, still-rendered message whose chunk vectors
|
|
3440
|
+
* were built over `enc:v3:` ciphertext (sdk#864) and have been discarded
|
|
3441
|
+
* rather than re-embedded at the user's own expense (client#5618). Skipped by
|
|
3442
|
+
* the embedding sweep so nothing re-embeds it, but NOT hidden — only
|
|
3443
|
+
* `tool_result` hides a row, and `isToolResultsRow` is what draws that line.
|
|
3433
3444
|
* - undefined/null: a normal message, or any row written before v44.
|
|
3434
3445
|
*
|
|
3435
3446
|
* Stored in the plaintext `origin` column — never encrypted. The deferred
|
|
@@ -3437,7 +3448,7 @@ type MessageFeedback = "like" | "dislike" | null;
|
|
|
3437
3448
|
* encrypted flag would be unreadable exactly where it matters and the skip
|
|
3438
3449
|
* would fail open.
|
|
3439
3450
|
*/
|
|
3440
|
-
type MessageOrigin = "tool_result";
|
|
3451
|
+
type MessageOrigin = "tool_result" | "chunks_discarded";
|
|
3441
3452
|
/**
|
|
3442
3453
|
* Metadata for files attached to messages.
|
|
3443
3454
|
*
|
|
@@ -3633,7 +3644,18 @@ interface StoredConversationSummary {
|
|
|
3633
3644
|
interface MessageChunk {
|
|
3634
3645
|
/** The chunk text */
|
|
3635
3646
|
text: string;
|
|
3636
|
-
/**
|
|
3647
|
+
/**
|
|
3648
|
+
* Embedding vector for this chunk.
|
|
3649
|
+
*
|
|
3650
|
+
* Read it through `decodeChunkVector` rather than passing it straight to a
|
|
3651
|
+
* numeric call site. This stays `number[]` only while the writer still emits
|
|
3652
|
+
* JSON arrays; it widens to `number[] | string` in the release that flips the
|
|
3653
|
+
* writer to base64 float32 (sdk#862), and that widening is the whole reason
|
|
3654
|
+
* that release is a major. Readers already accept both, so code written
|
|
3655
|
+
* against the decoder today needs no change when it happens — code written
|
|
3656
|
+
* against the raw array degrades silently, because a string handed to
|
|
3657
|
+
* `Float32Array.from` yields one NaN per character rather than an error.
|
|
3658
|
+
*/
|
|
3637
3659
|
vector: number[];
|
|
3638
3660
|
/** Character offset where this chunk starts in the original message */
|
|
3639
3661
|
startOffset: number;
|
|
@@ -7373,6 +7395,17 @@ declare function embedAllMessages(ctx: StorageOperationsContext, options: Embedd
|
|
|
7373
7395
|
roles?: ("user" | "assistant")[];
|
|
7374
7396
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7375
7397
|
minContentLength?: number;
|
|
7398
|
+
/**
|
|
7399
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7400
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7401
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7402
|
+
* only — `tool_result` rows stay excluded.
|
|
7403
|
+
*
|
|
7404
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7405
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7406
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7407
|
+
*/
|
|
7408
|
+
reembedDiscarded?: boolean;
|
|
7376
7409
|
}): Promise<number>;
|
|
7377
7410
|
/**
|
|
7378
7411
|
* Chunk and embed a single message, storing chunk embeddings in the database.
|
|
@@ -7418,6 +7451,21 @@ declare function chunkAndEmbedAllMessages(ctx: StorageOperationsContext, options
|
|
|
7418
7451
|
rechunkExisting?: boolean;
|
|
7419
7452
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7420
7453
|
minContentLength?: number;
|
|
7454
|
+
/**
|
|
7455
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7456
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7457
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7458
|
+
* only — `tool_result` rows stay excluded.
|
|
7459
|
+
*
|
|
7460
|
+
* `rechunkExisting` cannot substitute for it: a discarded row has neither
|
|
7461
|
+
* chunks nor vector, so it never reaches that check and falls straight to the
|
|
7462
|
+
* origin gate.
|
|
7463
|
+
*
|
|
7464
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7465
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7466
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7467
|
+
*/
|
|
7468
|
+
reembedDiscarded?: boolean;
|
|
7421
7469
|
}): Promise<number>;
|
|
7422
7470
|
|
|
7423
7471
|
/**
|
package/dist/server/index.d.ts
CHANGED
|
@@ -3423,13 +3423,24 @@ type ChatRole = "user" | "assistant" | "system";
|
|
|
3423
3423
|
*/
|
|
3424
3424
|
type MessageFeedback = "like" | "dislike" | null;
|
|
3425
3425
|
/**
|
|
3426
|
-
*
|
|
3427
|
-
*
|
|
3428
|
-
*
|
|
3426
|
+
* Provenance for a row that needs handling the content cannot justify. Set at
|
|
3427
|
+
* write time by whoever produced or repaired the row; a union so further kinds
|
|
3428
|
+
* can be added without another column.
|
|
3429
|
+
*
|
|
3430
|
+
* Read it as an enum, never as a boolean. The values do NOT share a meaning:
|
|
3431
|
+
* `tool_result` marks a synthetic row that is both unindexed and hidden, while
|
|
3432
|
+
* `chunks_discarded` marks an ordinary typed message that is unindexed and still
|
|
3433
|
+
* rendered. Any predicate that collapses this to "has an origin" hides real
|
|
3434
|
+
* messages (see `isToolResultsRow`).
|
|
3429
3435
|
*
|
|
3430
3436
|
* - `tool_result`: the hidden `[Tool Execution Results]` row built from a
|
|
3431
3437
|
* turn's auto-executed tool results. Skipped by the embedding sweep (see
|
|
3432
|
-
* `memoryEngine/embeddings`).
|
|
3438
|
+
* `memoryEngine/embeddings`) AND hidden from the transcript.
|
|
3439
|
+
* - `chunks_discarded`: an ordinary, still-rendered message whose chunk vectors
|
|
3440
|
+
* were built over `enc:v3:` ciphertext (sdk#864) and have been discarded
|
|
3441
|
+
* rather than re-embedded at the user's own expense (client#5618). Skipped by
|
|
3442
|
+
* the embedding sweep so nothing re-embeds it, but NOT hidden — only
|
|
3443
|
+
* `tool_result` hides a row, and `isToolResultsRow` is what draws that line.
|
|
3433
3444
|
* - undefined/null: a normal message, or any row written before v44.
|
|
3434
3445
|
*
|
|
3435
3446
|
* Stored in the plaintext `origin` column — never encrypted. The deferred
|
|
@@ -3437,7 +3448,7 @@ type MessageFeedback = "like" | "dislike" | null;
|
|
|
3437
3448
|
* encrypted flag would be unreadable exactly where it matters and the skip
|
|
3438
3449
|
* would fail open.
|
|
3439
3450
|
*/
|
|
3440
|
-
type MessageOrigin = "tool_result";
|
|
3451
|
+
type MessageOrigin = "tool_result" | "chunks_discarded";
|
|
3441
3452
|
/**
|
|
3442
3453
|
* Metadata for files attached to messages.
|
|
3443
3454
|
*
|
|
@@ -3633,7 +3644,18 @@ interface StoredConversationSummary {
|
|
|
3633
3644
|
interface MessageChunk {
|
|
3634
3645
|
/** The chunk text */
|
|
3635
3646
|
text: string;
|
|
3636
|
-
/**
|
|
3647
|
+
/**
|
|
3648
|
+
* Embedding vector for this chunk.
|
|
3649
|
+
*
|
|
3650
|
+
* Read it through `decodeChunkVector` rather than passing it straight to a
|
|
3651
|
+
* numeric call site. This stays `number[]` only while the writer still emits
|
|
3652
|
+
* JSON arrays; it widens to `number[] | string` in the release that flips the
|
|
3653
|
+
* writer to base64 float32 (sdk#862), and that widening is the whole reason
|
|
3654
|
+
* that release is a major. Readers already accept both, so code written
|
|
3655
|
+
* against the decoder today needs no change when it happens — code written
|
|
3656
|
+
* against the raw array degrades silently, because a string handed to
|
|
3657
|
+
* `Float32Array.from` yields one NaN per character rather than an error.
|
|
3658
|
+
*/
|
|
3637
3659
|
vector: number[];
|
|
3638
3660
|
/** Character offset where this chunk starts in the original message */
|
|
3639
3661
|
startOffset: number;
|
|
@@ -7373,6 +7395,17 @@ declare function embedAllMessages(ctx: StorageOperationsContext, options: Embedd
|
|
|
7373
7395
|
roles?: ("user" | "assistant")[];
|
|
7374
7396
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7375
7397
|
minContentLength?: number;
|
|
7398
|
+
/**
|
|
7399
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7400
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7401
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7402
|
+
* only — `tool_result` rows stay excluded.
|
|
7403
|
+
*
|
|
7404
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7405
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7406
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7407
|
+
*/
|
|
7408
|
+
reembedDiscarded?: boolean;
|
|
7376
7409
|
}): Promise<number>;
|
|
7377
7410
|
/**
|
|
7378
7411
|
* Chunk and embed a single message, storing chunk embeddings in the database.
|
|
@@ -7418,6 +7451,21 @@ declare function chunkAndEmbedAllMessages(ctx: StorageOperationsContext, options
|
|
|
7418
7451
|
rechunkExisting?: boolean;
|
|
7419
7452
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7420
7453
|
minContentLength?: number;
|
|
7454
|
+
/**
|
|
7455
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7456
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7457
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7458
|
+
* only — `tool_result` rows stay excluded.
|
|
7459
|
+
*
|
|
7460
|
+
* `rechunkExisting` cannot substitute for it: a discarded row has neither
|
|
7461
|
+
* chunks nor vector, so it never reaches that check and falls straight to the
|
|
7462
|
+
* origin gate.
|
|
7463
|
+
*
|
|
7464
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7465
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7466
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7467
|
+
*/
|
|
7468
|
+
reembedDiscarded?: boolean;
|
|
7421
7469
|
}): Promise<number>;
|
|
7422
7470
|
|
|
7423
7471
|
/**
|
package/dist/server/index.mjs
CHANGED
|
@@ -43,6 +43,7 @@ import {
|
|
|
43
43
|
createVaultEmbeddingCache,
|
|
44
44
|
createVaultMemoriesBatchOp,
|
|
45
45
|
createVaultMemoryOp,
|
|
46
|
+
dataUrlToArrayBuffer,
|
|
46
47
|
decomposeQuery,
|
|
47
48
|
decryptData,
|
|
48
49
|
decryptDataBatch,
|
|
@@ -129,6 +130,7 @@ import {
|
|
|
129
130
|
summarizeObservationTrends,
|
|
130
131
|
supersedeVaultMemoryOp,
|
|
131
132
|
traverseGraphLane,
|
|
133
|
+
uint8ArrayToBase64,
|
|
132
134
|
updateConversationProjectOp,
|
|
133
135
|
updateConversationTitleOp,
|
|
134
136
|
updateMessageChunksOp,
|
|
@@ -139,7 +141,7 @@ import {
|
|
|
139
141
|
updateVaultMemoryOp,
|
|
140
142
|
upsertMessageOp,
|
|
141
143
|
vaultMemoryToStored
|
|
142
|
-
} from "./chunk-
|
|
144
|
+
} from "./chunk-BZEAYCBV.mjs";
|
|
143
145
|
import {
|
|
144
146
|
createStreamingDeAnonymizer,
|
|
145
147
|
notifyConsolidationFallback,
|
|
@@ -9125,42 +9127,6 @@ var WalletPoller = class {
|
|
|
9125
9127
|
}
|
|
9126
9128
|
};
|
|
9127
9129
|
|
|
9128
|
-
// src/lib/processors/encoding.ts
|
|
9129
|
-
var B64_ENCODE_CHUNK = 32768 * 3;
|
|
9130
|
-
async function dataUrlToArrayBuffer(dataUrl) {
|
|
9131
|
-
if (dataUrl.startsWith("blob:") || dataUrl.startsWith("http://") || dataUrl.startsWith("https://")) {
|
|
9132
|
-
const response = await fetch(dataUrl);
|
|
9133
|
-
return response.arrayBuffer();
|
|
9134
|
-
}
|
|
9135
|
-
const base64 = dataUrl.split(",")[1];
|
|
9136
|
-
if (typeof Buffer !== "undefined") {
|
|
9137
|
-
const buf = Buffer.from(base64, "base64");
|
|
9138
|
-
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
9139
|
-
}
|
|
9140
|
-
const binary = atob(base64);
|
|
9141
|
-
const buffer = new ArrayBuffer(binary.length);
|
|
9142
|
-
const view = new Uint8Array(buffer);
|
|
9143
|
-
for (let i = 0; i < binary.length; i++) {
|
|
9144
|
-
view[i] = binary.charCodeAt(i);
|
|
9145
|
-
}
|
|
9146
|
-
return buffer;
|
|
9147
|
-
}
|
|
9148
|
-
function uint8ArrayToBase64(data) {
|
|
9149
|
-
if (typeof Buffer !== "undefined") {
|
|
9150
|
-
return Buffer.from(data).toString("base64");
|
|
9151
|
-
}
|
|
9152
|
-
let base64 = "";
|
|
9153
|
-
for (let i = 0; i < data.length; i += B64_ENCODE_CHUNK) {
|
|
9154
|
-
const slice = data.subarray(i, i + B64_ENCODE_CHUNK);
|
|
9155
|
-
let binary = "";
|
|
9156
|
-
for (let j = 0; j < slice.length; j++) {
|
|
9157
|
-
binary += String.fromCharCode(slice[j]);
|
|
9158
|
-
}
|
|
9159
|
-
base64 += btoa(binary);
|
|
9160
|
-
}
|
|
9161
|
-
return base64;
|
|
9162
|
-
}
|
|
9163
|
-
|
|
9164
9130
|
// src/lib/processors/ExcelProcessor.ts
|
|
9165
9131
|
if (typeof process !== "undefined" && typeof process.umask !== "function") {
|
|
9166
9132
|
process.umask = (_mask) => 18;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anuma/sdk",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.5.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "./dist/index.cjs",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -272,8 +272,8 @@
|
|
|
272
272
|
"undici@>=7 <7.28.0": ">=7.28.0 <8.0.0",
|
|
273
273
|
"vite@>=7 <7.3.5": ">=7.3.5",
|
|
274
274
|
"esbuild@>=0.27.3 <0.28.1": ">=0.28.1",
|
|
275
|
-
"js-yaml@>=4 <4.
|
|
276
|
-
"js-yaml@>=3 <3.15.
|
|
275
|
+
"js-yaml@>=4 <4.3.1": ">=4.3.1 <5.0.0",
|
|
276
|
+
"js-yaml@>=3 <3.15.1": ">=3.15.1 <4.0.0",
|
|
277
277
|
"markdown-it@<14.2.0": ">=14.2.0",
|
|
278
278
|
"brace-expansion@1": ">=1.1.18 <2.0.0",
|
|
279
279
|
"brace-expansion@2": ">=2.1.4 <3.0.0"
|