@anuma/sdk 6.4.2 → 6.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/{chunk-WYARTNEN.mjs → chunk-4M6W7QLZ.mjs} +233 -8
- package/dist/{react/chunk-ILAWDQDE.mjs → expo/chunk-B5UPJRCU.mjs} +16 -1
- package/dist/expo/{chunk-DT6QNRZL.mjs → chunk-VOMDHMKH.mjs} +1 -1
- package/dist/{react/consolidate-AMPKH6YJ.mjs → expo/consolidate-TWE6OSSZ.mjs} +2 -2
- package/dist/expo/index.cjs +306 -156
- package/dist/expo/index.d.mts +163 -7
- package/dist/expo/index.d.ts +163 -7
- package/dist/expo/index.mjs +33 -154
- package/dist/expo/{recall-7FOWWX5T.mjs → recall-EZIHN5VU.mjs} +2 -2
- package/dist/index.cjs +71 -0
- package/dist/index.d.mts +121 -1
- package/dist/index.d.ts +121 -1
- package/dist/index.mjs +67 -0
- package/dist/react/{chunk-VMI74IUX.mjs → chunk-2ARMO2NN.mjs} +1 -1
- package/dist/{expo/chunk-ILAWDQDE.mjs → react/chunk-B5UPJRCU.mjs} +16 -1
- package/dist/react/{chunk-IO4ZCPK2.mjs → chunk-S6WPXD5N.mjs} +254 -8
- package/dist/{expo/consolidate-3MEWQNM3.mjs → react/consolidate-ZCGS7QUA.mjs} +2 -2
- package/dist/react/index.cjs +331 -206
- package/dist/react/index.d.mts +163 -7
- package/dist/react/index.d.ts +163 -7
- package/dist/react/index.mjs +36 -202
- package/dist/react/{recall-2J2755WC.mjs → recall-ROEK55FJ.mjs} +2 -2
- package/dist/server/{chunk-KF4HV3QE.mjs → chunk-CR5P3YIK.mjs} +1 -1
- package/dist/server/{chunk-EKYGWD5N.mjs → chunk-TD7SMY7C.mjs} +98 -8
- package/dist/server/{chunk-OBDRKQQI.mjs → chunk-VHWLLYKO.mjs} +16 -1
- package/dist/server/{consolidate-I6TOH7CB.mjs → consolidate-FWYK5KXB.mjs} +2 -2
- package/dist/server/index.cjs +158 -48
- package/dist/server/index.d.mts +54 -6
- package/dist/server/index.d.ts +54 -6
- package/dist/server/index.mjs +24 -43
- package/dist/server/{recall-IVV6EILK.mjs → recall-CGDEW2LQ.mjs} +2 -2
- package/package.json +1 -1
package/dist/server/index.cjs
CHANGED
|
@@ -15911,6 +15911,90 @@ var init_vector = __esm({
|
|
|
15911
15911
|
}
|
|
15912
15912
|
});
|
|
15913
15913
|
|
|
15914
|
+
// src/lib/processors/encoding.ts
|
|
15915
|
+
async function dataUrlToArrayBuffer(dataUrl) {
|
|
15916
|
+
if (dataUrl.startsWith("blob:") || dataUrl.startsWith("http://") || dataUrl.startsWith("https://")) {
|
|
15917
|
+
const response = await fetch(dataUrl);
|
|
15918
|
+
return response.arrayBuffer();
|
|
15919
|
+
}
|
|
15920
|
+
const base64 = dataUrl.split(",")[1];
|
|
15921
|
+
if (typeof Buffer !== "undefined") {
|
|
15922
|
+
const buf = Buffer.from(base64, "base64");
|
|
15923
|
+
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
15924
|
+
}
|
|
15925
|
+
const binary = atob(base64);
|
|
15926
|
+
const buffer = new ArrayBuffer(binary.length);
|
|
15927
|
+
const view = new Uint8Array(buffer);
|
|
15928
|
+
for (let i = 0; i < binary.length; i++) {
|
|
15929
|
+
view[i] = binary.charCodeAt(i);
|
|
15930
|
+
}
|
|
15931
|
+
return buffer;
|
|
15932
|
+
}
|
|
15933
|
+
function uint8ArrayToBase64(data) {
|
|
15934
|
+
if (typeof Buffer !== "undefined") {
|
|
15935
|
+
return Buffer.from(data).toString("base64");
|
|
15936
|
+
}
|
|
15937
|
+
let base64 = "";
|
|
15938
|
+
for (let i = 0; i < data.length; i += B64_ENCODE_CHUNK) {
|
|
15939
|
+
const slice = data.subarray(i, i + B64_ENCODE_CHUNK);
|
|
15940
|
+
let binary = "";
|
|
15941
|
+
for (let j = 0; j < slice.length; j++) {
|
|
15942
|
+
binary += String.fromCharCode(slice[j]);
|
|
15943
|
+
}
|
|
15944
|
+
base64 += btoa(binary);
|
|
15945
|
+
}
|
|
15946
|
+
return base64;
|
|
15947
|
+
}
|
|
15948
|
+
function base64ToUint8Array(base64) {
|
|
15949
|
+
if (typeof Buffer !== "undefined") {
|
|
15950
|
+
const buf = Buffer.from(base64, "base64");
|
|
15951
|
+
return new Uint8Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
|
|
15952
|
+
}
|
|
15953
|
+
const binary = atob(base64);
|
|
15954
|
+
const bytes = new Uint8Array(binary.length);
|
|
15955
|
+
for (let i = 0; i < binary.length; i++) {
|
|
15956
|
+
bytes[i] = binary.charCodeAt(i);
|
|
15957
|
+
}
|
|
15958
|
+
return bytes;
|
|
15959
|
+
}
|
|
15960
|
+
var B64_ENCODE_CHUNK;
|
|
15961
|
+
var init_encoding = __esm({
|
|
15962
|
+
"src/lib/processors/encoding.ts"() {
|
|
15963
|
+
"use strict";
|
|
15964
|
+
B64_ENCODE_CHUNK = 32768 * 3;
|
|
15965
|
+
}
|
|
15966
|
+
});
|
|
15967
|
+
|
|
15968
|
+
// src/lib/memoryEngine/vectorEncoding.ts
|
|
15969
|
+
function decodeChunkVector(vector, onMalformed) {
|
|
15970
|
+
if (!vector || vector.length === 0) return new Float32Array(0);
|
|
15971
|
+
if (typeof vector !== "string") return Float32Array.from(vector);
|
|
15972
|
+
if (vector.length % 4 !== 0 || !CANONICAL_BASE64.test(vector)) {
|
|
15973
|
+
onMalformed?.();
|
|
15974
|
+
return new Float32Array(0);
|
|
15975
|
+
}
|
|
15976
|
+
try {
|
|
15977
|
+
const bytes = base64ToUint8Array(vector);
|
|
15978
|
+
if (bytes.byteLength === 0 || bytes.byteLength % BYTES_PER_FLOAT32 !== 0) {
|
|
15979
|
+
onMalformed?.();
|
|
15980
|
+
return new Float32Array(0);
|
|
15981
|
+
}
|
|
15982
|
+
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / BYTES_PER_FLOAT32);
|
|
15983
|
+
} catch {
|
|
15984
|
+
onMalformed?.();
|
|
15985
|
+
return new Float32Array(0);
|
|
15986
|
+
}
|
|
15987
|
+
}
|
|
15988
|
+
var BYTES_PER_FLOAT32, CANONICAL_BASE64;
|
|
15989
|
+
var init_vectorEncoding = __esm({
|
|
15990
|
+
"src/lib/memoryEngine/vectorEncoding.ts"() {
|
|
15991
|
+
"use strict";
|
|
15992
|
+
init_encoding();
|
|
15993
|
+
BYTES_PER_FLOAT32 = 4;
|
|
15994
|
+
CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
|
|
15995
|
+
}
|
|
15996
|
+
});
|
|
15997
|
+
|
|
15914
15998
|
// src/lib/db/chat/conversationEncryption.ts
|
|
15915
15999
|
async function encryptConversationFields(conversation, address, signMessage, embeddedWalletSigner) {
|
|
15916
16000
|
if (!address || !signMessage) {
|
|
@@ -16797,6 +16881,7 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16797
16881
|
const messages = await ctx.messagesCollection.query(...queryConditions).fetch();
|
|
16798
16882
|
const candidates = [];
|
|
16799
16883
|
let staleSkipped = 0;
|
|
16884
|
+
let unreadableVectors = 0;
|
|
16800
16885
|
for (const message of messages) {
|
|
16801
16886
|
const msgConvId = String(message._getRaw("conversation_id"));
|
|
16802
16887
|
if (!activeConversationIds.has(msgConvId)) continue;
|
|
@@ -16834,7 +16919,9 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16834
16919
|
const chunks = await readJsonField(message, "chunks", ctx.walletAddress);
|
|
16835
16920
|
if (chunks && chunks.length > 0) {
|
|
16836
16921
|
const vectors = chunks.map(
|
|
16837
|
-
(chunk) => chunk.vector
|
|
16922
|
+
(chunk) => decodeChunkVector(chunk.vector, () => {
|
|
16923
|
+
unreadableVectors++;
|
|
16924
|
+
})
|
|
16838
16925
|
);
|
|
16839
16926
|
chunkCache?.set(message.id, { version, chunks: vectors });
|
|
16840
16927
|
for (let ci = 0; ci < chunks.length; ci++) {
|
|
@@ -16865,6 +16952,11 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16865
16952
|
`searchChunksOp: skipped ${staleSkipped} messages whose embedding model differs from the current model (${embeddingModel}) \u2014 re-embed via chunkAndEmbedAllMessages`
|
|
16866
16953
|
);
|
|
16867
16954
|
}
|
|
16955
|
+
if (unreadableVectors > 0) {
|
|
16956
|
+
getLogger().warn(
|
|
16957
|
+
`searchChunksOp: could not read ${unreadableVectors} chunk vectors and left them out of ranking \u2014 the stored value is corrupt, so those chunks stay unsearchable until the message is re-embedded via chunkAndEmbedAllMessages`
|
|
16958
|
+
);
|
|
16959
|
+
}
|
|
16868
16960
|
const topK = candidates.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
|
|
16869
16961
|
const uniqueMessages = /* @__PURE__ */ new Map();
|
|
16870
16962
|
for (const c of topK) uniqueMessages.set(c.message.id, c.message);
|
|
@@ -16986,6 +17078,7 @@ var init_operations = __esm({
|
|
|
16986
17078
|
import_uuid2 = require("uuid");
|
|
16987
17079
|
init_logger();
|
|
16988
17080
|
init_vector();
|
|
17081
|
+
init_vectorEncoding();
|
|
16989
17082
|
init_encryption_utils();
|
|
16990
17083
|
init_conversationEncryption();
|
|
16991
17084
|
init_encryption();
|
|
@@ -17025,6 +17118,20 @@ var init_clientConfig = __esm({
|
|
|
17025
17118
|
}
|
|
17026
17119
|
});
|
|
17027
17120
|
|
|
17121
|
+
// src/lib/internalFlowMarker.ts
|
|
17122
|
+
function withInternalFlowMarker(systemPrompt) {
|
|
17123
|
+
if (systemPrompt.includes(INTERNAL_FLOW_MARKER)) return systemPrompt;
|
|
17124
|
+
return `${INTERNAL_FLOW_MARKER}
|
|
17125
|
+
${systemPrompt}`;
|
|
17126
|
+
}
|
|
17127
|
+
var INTERNAL_FLOW_MARKER;
|
|
17128
|
+
var init_internalFlowMarker = __esm({
|
|
17129
|
+
"src/lib/internalFlowMarker.ts"() {
|
|
17130
|
+
"use strict";
|
|
17131
|
+
INTERNAL_FLOW_MARKER = "Anuma internal first-party flow (not user chat).";
|
|
17132
|
+
}
|
|
17133
|
+
});
|
|
17134
|
+
|
|
17028
17135
|
// src/lib/db/entities/types.ts
|
|
17029
17136
|
function normalizeEntityName(name) {
|
|
17030
17137
|
return name.trim().toLowerCase();
|
|
@@ -18657,6 +18764,15 @@ var init_chunking = __esm({
|
|
|
18657
18764
|
}
|
|
18658
18765
|
});
|
|
18659
18766
|
|
|
18767
|
+
// src/lib/chat/toolResults.ts
|
|
18768
|
+
var TOOL_RESULT_ORIGIN;
|
|
18769
|
+
var init_toolResults = __esm({
|
|
18770
|
+
"src/lib/chat/toolResults.ts"() {
|
|
18771
|
+
"use strict";
|
|
18772
|
+
TOOL_RESULT_ORIGIN = "tool_result";
|
|
18773
|
+
}
|
|
18774
|
+
});
|
|
18775
|
+
|
|
18660
18776
|
// src/lib/memoryEngine/constants.ts
|
|
18661
18777
|
var DEFAULT_API_EMBEDDING_MODEL;
|
|
18662
18778
|
var init_constants = __esm({
|
|
@@ -19742,6 +19858,11 @@ var init_generate = __esm({
|
|
|
19742
19858
|
});
|
|
19743
19859
|
|
|
19744
19860
|
// src/lib/memoryEngine/embeddings.ts
|
|
19861
|
+
function isNonEmbeddableOrigin(origin, reembedDiscarded = false) {
|
|
19862
|
+
if (typeof origin !== "string") return false;
|
|
19863
|
+
if (reembedDiscarded && origin === CHUNKS_DISCARDED_ORIGIN) return false;
|
|
19864
|
+
return NON_EMBEDDABLE_ORIGINS.has(origin);
|
|
19865
|
+
}
|
|
19745
19866
|
async function embedMessage(ctx, messageId, options) {
|
|
19746
19867
|
const message = await getMessageOp(ctx, messageId);
|
|
19747
19868
|
if (!message) {
|
|
@@ -19750,7 +19871,7 @@ async function embedMessage(ctx, messageId, options) {
|
|
|
19750
19871
|
if (message.vector && message.vector.length > 0) {
|
|
19751
19872
|
return message;
|
|
19752
19873
|
}
|
|
19753
|
-
if (message.origin
|
|
19874
|
+
if (isNonEmbeddableOrigin(message.origin)) {
|
|
19754
19875
|
return message;
|
|
19755
19876
|
}
|
|
19756
19877
|
if (isEncrypted(message.content)) {
|
|
@@ -19786,7 +19907,7 @@ async function embedAllMessages(ctx, options, filter) {
|
|
|
19786
19907
|
if (message.role === "system") {
|
|
19787
19908
|
continue;
|
|
19788
19909
|
}
|
|
19789
|
-
if (message.origin
|
|
19910
|
+
if (isNonEmbeddableOrigin(message.origin, filter?.reembedDiscarded)) {
|
|
19790
19911
|
continue;
|
|
19791
19912
|
}
|
|
19792
19913
|
considered++;
|
|
@@ -19826,7 +19947,7 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
|
|
|
19826
19947
|
if (message.chunks && message.chunks.length > 0) {
|
|
19827
19948
|
return message;
|
|
19828
19949
|
}
|
|
19829
|
-
if (message.origin
|
|
19950
|
+
if (isNonEmbeddableOrigin(message.origin)) {
|
|
19830
19951
|
return message;
|
|
19831
19952
|
}
|
|
19832
19953
|
if (isEncrypted(message.content)) {
|
|
@@ -19874,7 +19995,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
|
|
|
19874
19995
|
if (hasVector && !filter?.rechunkExisting && !isStale) continue;
|
|
19875
19996
|
if (filter?.roles && !filter.roles.includes(message.role)) continue;
|
|
19876
19997
|
if (message.role === "system") continue;
|
|
19877
|
-
if (message.origin
|
|
19998
|
+
if (isNonEmbeddableOrigin(message.origin, filter?.reembedDiscarded)) continue;
|
|
19878
19999
|
considered++;
|
|
19879
20000
|
if (isEncrypted(message.content)) {
|
|
19880
20001
|
stillEncrypted++;
|
|
@@ -19943,10 +20064,11 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
|
|
|
19943
20064
|
}
|
|
19944
20065
|
return embeddedCount;
|
|
19945
20066
|
}
|
|
19946
|
-
var DEFAULT_MIN_CONTENT_LENGTH,
|
|
20067
|
+
var DEFAULT_MIN_CONTENT_LENGTH, CHUNKS_DISCARDED_ORIGIN, NON_EMBEDDABLE_ORIGINS;
|
|
19947
20068
|
var init_embeddings = __esm({
|
|
19948
20069
|
"src/lib/memoryEngine/embeddings.ts"() {
|
|
19949
20070
|
"use strict";
|
|
20071
|
+
init_toolResults();
|
|
19950
20072
|
init_operations();
|
|
19951
20073
|
init_encryption_utils();
|
|
19952
20074
|
init_logger();
|
|
@@ -19955,7 +20077,8 @@ var init_embeddings = __esm({
|
|
|
19955
20077
|
init_generate();
|
|
19956
20078
|
init_generate();
|
|
19957
20079
|
DEFAULT_MIN_CONTENT_LENGTH = 10;
|
|
19958
|
-
|
|
20080
|
+
CHUNKS_DISCARDED_ORIGIN = "chunks_discarded";
|
|
20081
|
+
NON_EMBEDDABLE_ORIGINS = /* @__PURE__ */ new Set([TOOL_RESULT_ORIGIN, CHUNKS_DISCARDED_ORIGIN]);
|
|
19959
20082
|
}
|
|
19960
20083
|
});
|
|
19961
20084
|
|
|
@@ -20538,7 +20661,12 @@ async function attemptPortalJson(req, endpoint) {
|
|
|
20538
20661
|
const isAnthropic = req.model.startsWith("anthropic/");
|
|
20539
20662
|
const prefill = isAnthropic ? "{" : "";
|
|
20540
20663
|
const messages = [
|
|
20541
|
-
|
|
20664
|
+
// Every caller of this helper is a first-party BACKGROUND op (fact extraction,
|
|
20665
|
+
// topic sweep, consolidation, classifiers, graph traversal, query
|
|
20666
|
+
// decomposition). Marking here — the one place they all funnel through — is what
|
|
20667
|
+
// keeps the portal's detector reading them as genuine rather than markerless.
|
|
20668
|
+
// See ../internalFlowMarker.
|
|
20669
|
+
{ role: "system", content: withInternalFlowMarker(req.systemPrompt) },
|
|
20542
20670
|
{ role: "user", content: req.userMessage }
|
|
20543
20671
|
];
|
|
20544
20672
|
if (prefill) messages.push({ role: "assistant", content: prefill });
|
|
@@ -20676,6 +20804,7 @@ var init_portalLlm = __esm({
|
|
|
20676
20804
|
"use strict";
|
|
20677
20805
|
init_clientConfig();
|
|
20678
20806
|
init_endpointOverride();
|
|
20807
|
+
init_internalFlowMarker();
|
|
20679
20808
|
init_logger();
|
|
20680
20809
|
RESPONSE_FORMAT_OK = /* @__PURE__ */ new Set(["openai", "inclusionai", "deepseek"]);
|
|
20681
20810
|
RESPONSE_SCHEMA_OK = /* @__PURE__ */ new Set(["openai"]);
|
|
@@ -26617,6 +26746,7 @@ init_types2();
|
|
|
26617
26746
|
|
|
26618
26747
|
// src/lib/chat/summarize.ts
|
|
26619
26748
|
init_clientConfig();
|
|
26749
|
+
init_internalFlowMarker();
|
|
26620
26750
|
init_logger();
|
|
26621
26751
|
var DEFAULT_SUMMARY_TOKEN_THRESHOLD = 4e3;
|
|
26622
26752
|
var DEFAULT_SUMMARY_MIN_WINDOW_MESSAGES = 4;
|
|
@@ -26809,7 +26939,17 @@ async function callSummarizationLlm(prompt, model, token, baseUrl) {
|
|
|
26809
26939
|
body: JSON.stringify({
|
|
26810
26940
|
model,
|
|
26811
26941
|
stream: false,
|
|
26812
|
-
|
|
26942
|
+
// The marker rides a system message because this call has no other
|
|
26943
|
+
// provenance: it posts to the MAIN chat endpoint (not /utility/*), carries no
|
|
26944
|
+
// conversationId, and its prompt is neither the app's chat base prompt nor any
|
|
26945
|
+
// registered flow fingerprint — so without this it reads as markerless, i.e.
|
|
26946
|
+
// as a scripted abuser. Progressive summarization runs for every free-tier
|
|
26947
|
+
// long conversation, so at the PORTAL_DETECTION_REJECT_MARKERLESS flip it
|
|
26948
|
+
// would 403 and silently fall back to verbatim history (context bloat + cost).
|
|
26949
|
+
messages: [
|
|
26950
|
+
{ role: "system", content: INTERNAL_FLOW_MARKER },
|
|
26951
|
+
{ role: "user", content: prompt }
|
|
26952
|
+
]
|
|
26813
26953
|
}),
|
|
26814
26954
|
signal: controller.signal
|
|
26815
26955
|
});
|
|
@@ -30496,6 +30636,7 @@ init_reranker();
|
|
|
30496
30636
|
|
|
30497
30637
|
// src/lib/memory/synthesizeProfile.ts
|
|
30498
30638
|
init_operations3();
|
|
30639
|
+
init_internalFlowMarker();
|
|
30499
30640
|
init_logger();
|
|
30500
30641
|
init_constants();
|
|
30501
30642
|
init_embeddings();
|
|
@@ -30857,7 +30998,10 @@ async function synthesizeFacet(facet, ctx, options, prior) {
|
|
|
30857
30998
|
limit,
|
|
30858
30999
|
types: ["fact"],
|
|
30859
31000
|
maxTokens: DEFAULT_FACET_MAX_TOKENS,
|
|
30860
|
-
|
|
31001
|
+
// Profile-facet synthesis is a background op, so it marks its own prompt.
|
|
31002
|
+
// reflect() itself must NOT mark — it also serves the user's own questions,
|
|
31003
|
+
// and that traffic is chat, not an internal flow. See ../internalFlowMarker.
|
|
31004
|
+
systemPrompt: withInternalFlowMarker(buildFacetSystemPrompt(facet)),
|
|
30861
31005
|
responseSchema: facetResponseSchema(facet.key),
|
|
30862
31006
|
memories
|
|
30863
31007
|
});
|
|
@@ -32619,44 +32763,7 @@ var WalletPoller = class {
|
|
|
32619
32763
|
|
|
32620
32764
|
// src/lib/processors/ExcelProcessor.ts
|
|
32621
32765
|
init_logger();
|
|
32622
|
-
|
|
32623
|
-
// src/lib/processors/encoding.ts
|
|
32624
|
-
var B64_ENCODE_CHUNK = 32768 * 3;
|
|
32625
|
-
async function dataUrlToArrayBuffer(dataUrl) {
|
|
32626
|
-
if (dataUrl.startsWith("blob:") || dataUrl.startsWith("http://") || dataUrl.startsWith("https://")) {
|
|
32627
|
-
const response = await fetch(dataUrl);
|
|
32628
|
-
return response.arrayBuffer();
|
|
32629
|
-
}
|
|
32630
|
-
const base64 = dataUrl.split(",")[1];
|
|
32631
|
-
if (typeof Buffer !== "undefined") {
|
|
32632
|
-
const buf = Buffer.from(base64, "base64");
|
|
32633
|
-
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
32634
|
-
}
|
|
32635
|
-
const binary = atob(base64);
|
|
32636
|
-
const buffer = new ArrayBuffer(binary.length);
|
|
32637
|
-
const view = new Uint8Array(buffer);
|
|
32638
|
-
for (let i = 0; i < binary.length; i++) {
|
|
32639
|
-
view[i] = binary.charCodeAt(i);
|
|
32640
|
-
}
|
|
32641
|
-
return buffer;
|
|
32642
|
-
}
|
|
32643
|
-
function uint8ArrayToBase64(data) {
|
|
32644
|
-
if (typeof Buffer !== "undefined") {
|
|
32645
|
-
return Buffer.from(data).toString("base64");
|
|
32646
|
-
}
|
|
32647
|
-
let base64 = "";
|
|
32648
|
-
for (let i = 0; i < data.length; i += B64_ENCODE_CHUNK) {
|
|
32649
|
-
const slice = data.subarray(i, i + B64_ENCODE_CHUNK);
|
|
32650
|
-
let binary = "";
|
|
32651
|
-
for (let j = 0; j < slice.length; j++) {
|
|
32652
|
-
binary += String.fromCharCode(slice[j]);
|
|
32653
|
-
}
|
|
32654
|
-
base64 += btoa(binary);
|
|
32655
|
-
}
|
|
32656
|
-
return base64;
|
|
32657
|
-
}
|
|
32658
|
-
|
|
32659
|
-
// src/lib/processors/ExcelProcessor.ts
|
|
32766
|
+
init_encoding();
|
|
32660
32767
|
if (typeof process !== "undefined" && typeof process.umask !== "function") {
|
|
32661
32768
|
process.umask = (_mask) => 18;
|
|
32662
32769
|
}
|
|
@@ -32835,6 +32942,7 @@ var PdfProcessor = class {
|
|
|
32835
32942
|
|
|
32836
32943
|
// src/lib/processors/TextProcessor.ts
|
|
32837
32944
|
init_logger();
|
|
32945
|
+
init_encoding();
|
|
32838
32946
|
var MIME = {
|
|
32839
32947
|
PLAIN: "text/plain",
|
|
32840
32948
|
MARKDOWN: "text/markdown",
|
|
@@ -32918,6 +33026,7 @@ var TextProcessor = class {
|
|
|
32918
33026
|
// src/lib/processors/WordProcessor.ts
|
|
32919
33027
|
var import_mammoth = __toESM(require("mammoth"));
|
|
32920
33028
|
init_logger();
|
|
33029
|
+
init_encoding();
|
|
32921
33030
|
var WordProcessor = class {
|
|
32922
33031
|
constructor() {
|
|
32923
33032
|
this.name = "word";
|
|
@@ -32961,6 +33070,7 @@ var WordProcessor = class {
|
|
|
32961
33070
|
// src/lib/processors/ZipProcessor.ts
|
|
32962
33071
|
var import_jszip = __toESM(require("jszip"));
|
|
32963
33072
|
init_logger();
|
|
33073
|
+
init_encoding();
|
|
32964
33074
|
var ZipProcessor = class {
|
|
32965
33075
|
constructor(options = {}) {
|
|
32966
33076
|
this.name = "zip";
|
package/dist/server/index.d.mts
CHANGED
|
@@ -3423,13 +3423,24 @@ type ChatRole = "user" | "assistant" | "system";
|
|
|
3423
3423
|
*/
|
|
3424
3424
|
type MessageFeedback = "like" | "dislike" | null;
|
|
3425
3425
|
/**
|
|
3426
|
-
*
|
|
3427
|
-
*
|
|
3428
|
-
*
|
|
3426
|
+
* Provenance for a row that needs handling the content cannot justify. Set at
|
|
3427
|
+
* write time by whoever produced or repaired the row; a union so further kinds
|
|
3428
|
+
* can be added without another column.
|
|
3429
|
+
*
|
|
3430
|
+
* Read it as an enum, never as a boolean. The values do NOT share a meaning:
|
|
3431
|
+
* `tool_result` marks a synthetic row that is both unindexed and hidden, while
|
|
3432
|
+
* `chunks_discarded` marks an ordinary typed message that is unindexed and still
|
|
3433
|
+
* rendered. Any predicate that collapses this to "has an origin" hides real
|
|
3434
|
+
* messages (see `isToolResultsRow`).
|
|
3429
3435
|
*
|
|
3430
3436
|
* - `tool_result`: the hidden `[Tool Execution Results]` row built from a
|
|
3431
3437
|
* turn's auto-executed tool results. Skipped by the embedding sweep (see
|
|
3432
|
-
* `memoryEngine/embeddings`).
|
|
3438
|
+
* `memoryEngine/embeddings`) AND hidden from the transcript.
|
|
3439
|
+
* - `chunks_discarded`: an ordinary, still-rendered message whose chunk vectors
|
|
3440
|
+
* were built over `enc:v3:` ciphertext (sdk#864) and have been discarded
|
|
3441
|
+
* rather than re-embedded at the user's own expense (client#5618). Skipped by
|
|
3442
|
+
* the embedding sweep so nothing re-embeds it, but NOT hidden — only
|
|
3443
|
+
* `tool_result` hides a row, and `isToolResultsRow` is what draws that line.
|
|
3433
3444
|
* - undefined/null: a normal message, or any row written before v44.
|
|
3434
3445
|
*
|
|
3435
3446
|
* Stored in the plaintext `origin` column — never encrypted. The deferred
|
|
@@ -3437,7 +3448,7 @@ type MessageFeedback = "like" | "dislike" | null;
|
|
|
3437
3448
|
* encrypted flag would be unreadable exactly where it matters and the skip
|
|
3438
3449
|
* would fail open.
|
|
3439
3450
|
*/
|
|
3440
|
-
type MessageOrigin = "tool_result";
|
|
3451
|
+
type MessageOrigin = "tool_result" | "chunks_discarded";
|
|
3441
3452
|
/**
|
|
3442
3453
|
* Metadata for files attached to messages.
|
|
3443
3454
|
*
|
|
@@ -3633,7 +3644,18 @@ interface StoredConversationSummary {
|
|
|
3633
3644
|
interface MessageChunk {
|
|
3634
3645
|
/** The chunk text */
|
|
3635
3646
|
text: string;
|
|
3636
|
-
/**
|
|
3647
|
+
/**
|
|
3648
|
+
* Embedding vector for this chunk.
|
|
3649
|
+
*
|
|
3650
|
+
* Read it through `decodeChunkVector` rather than passing it straight to a
|
|
3651
|
+
* numeric call site. This stays `number[]` only while the writer still emits
|
|
3652
|
+
* JSON arrays; it widens to `number[] | string` in the release that flips the
|
|
3653
|
+
* writer to base64 float32 (sdk#862), and that widening is the whole reason
|
|
3654
|
+
* that release is a major. Readers already accept both, so code written
|
|
3655
|
+
* against the decoder today needs no change when it happens — code written
|
|
3656
|
+
* against the raw array degrades silently, because a string handed to
|
|
3657
|
+
* `Float32Array.from` yields one NaN per character rather than an error.
|
|
3658
|
+
*/
|
|
3637
3659
|
vector: number[];
|
|
3638
3660
|
/** Character offset where this chunk starts in the original message */
|
|
3639
3661
|
startOffset: number;
|
|
@@ -7373,6 +7395,17 @@ declare function embedAllMessages(ctx: StorageOperationsContext, options: Embedd
|
|
|
7373
7395
|
roles?: ("user" | "assistant")[];
|
|
7374
7396
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7375
7397
|
minContentLength?: number;
|
|
7398
|
+
/**
|
|
7399
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7400
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7401
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7402
|
+
* only — `tool_result` rows stay excluded.
|
|
7403
|
+
*
|
|
7404
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7405
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7406
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7407
|
+
*/
|
|
7408
|
+
reembedDiscarded?: boolean;
|
|
7376
7409
|
}): Promise<number>;
|
|
7377
7410
|
/**
|
|
7378
7411
|
* Chunk and embed a single message, storing chunk embeddings in the database.
|
|
@@ -7418,6 +7451,21 @@ declare function chunkAndEmbedAllMessages(ctx: StorageOperationsContext, options
|
|
|
7418
7451
|
rechunkExisting?: boolean;
|
|
7419
7452
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7420
7453
|
minContentLength?: number;
|
|
7454
|
+
/**
|
|
7455
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7456
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7457
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7458
|
+
* only — `tool_result` rows stay excluded.
|
|
7459
|
+
*
|
|
7460
|
+
* `rechunkExisting` cannot substitute for it: a discarded row has neither
|
|
7461
|
+
* chunks nor vector, so it never reaches that check and falls straight to the
|
|
7462
|
+
* origin gate.
|
|
7463
|
+
*
|
|
7464
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7465
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7466
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7467
|
+
*/
|
|
7468
|
+
reembedDiscarded?: boolean;
|
|
7421
7469
|
}): Promise<number>;
|
|
7422
7470
|
|
|
7423
7471
|
/**
|
package/dist/server/index.d.ts
CHANGED
|
@@ -3423,13 +3423,24 @@ type ChatRole = "user" | "assistant" | "system";
|
|
|
3423
3423
|
*/
|
|
3424
3424
|
type MessageFeedback = "like" | "dislike" | null;
|
|
3425
3425
|
/**
|
|
3426
|
-
*
|
|
3427
|
-
*
|
|
3428
|
-
*
|
|
3426
|
+
* Provenance for a row that needs handling the content cannot justify. Set at
|
|
3427
|
+
* write time by whoever produced or repaired the row; a union so further kinds
|
|
3428
|
+
* can be added without another column.
|
|
3429
|
+
*
|
|
3430
|
+
* Read it as an enum, never as a boolean. The values do NOT share a meaning:
|
|
3431
|
+
* `tool_result` marks a synthetic row that is both unindexed and hidden, while
|
|
3432
|
+
* `chunks_discarded` marks an ordinary typed message that is unindexed and still
|
|
3433
|
+
* rendered. Any predicate that collapses this to "has an origin" hides real
|
|
3434
|
+
* messages (see `isToolResultsRow`).
|
|
3429
3435
|
*
|
|
3430
3436
|
* - `tool_result`: the hidden `[Tool Execution Results]` row built from a
|
|
3431
3437
|
* turn's auto-executed tool results. Skipped by the embedding sweep (see
|
|
3432
|
-
* `memoryEngine/embeddings`).
|
|
3438
|
+
* `memoryEngine/embeddings`) AND hidden from the transcript.
|
|
3439
|
+
* - `chunks_discarded`: an ordinary, still-rendered message whose chunk vectors
|
|
3440
|
+
* were built over `enc:v3:` ciphertext (sdk#864) and have been discarded
|
|
3441
|
+
* rather than re-embedded at the user's own expense (client#5618). Skipped by
|
|
3442
|
+
* the embedding sweep so nothing re-embeds it, but NOT hidden — only
|
|
3443
|
+
* `tool_result` hides a row, and `isToolResultsRow` is what draws that line.
|
|
3433
3444
|
* - undefined/null: a normal message, or any row written before v44.
|
|
3434
3445
|
*
|
|
3435
3446
|
* Stored in the plaintext `origin` column — never encrypted. The deferred
|
|
@@ -3437,7 +3448,7 @@ type MessageFeedback = "like" | "dislike" | null;
|
|
|
3437
3448
|
* encrypted flag would be unreadable exactly where it matters and the skip
|
|
3438
3449
|
* would fail open.
|
|
3439
3450
|
*/
|
|
3440
|
-
type MessageOrigin = "tool_result";
|
|
3451
|
+
type MessageOrigin = "tool_result" | "chunks_discarded";
|
|
3441
3452
|
/**
|
|
3442
3453
|
* Metadata for files attached to messages.
|
|
3443
3454
|
*
|
|
@@ -3633,7 +3644,18 @@ interface StoredConversationSummary {
|
|
|
3633
3644
|
interface MessageChunk {
|
|
3634
3645
|
/** The chunk text */
|
|
3635
3646
|
text: string;
|
|
3636
|
-
/**
|
|
3647
|
+
/**
|
|
3648
|
+
* Embedding vector for this chunk.
|
|
3649
|
+
*
|
|
3650
|
+
* Read it through `decodeChunkVector` rather than passing it straight to a
|
|
3651
|
+
* numeric call site. This stays `number[]` only while the writer still emits
|
|
3652
|
+
* JSON arrays; it widens to `number[] | string` in the release that flips the
|
|
3653
|
+
* writer to base64 float32 (sdk#862), and that widening is the whole reason
|
|
3654
|
+
* that release is a major. Readers already accept both, so code written
|
|
3655
|
+
* against the decoder today needs no change when it happens — code written
|
|
3656
|
+
* against the raw array degrades silently, because a string handed to
|
|
3657
|
+
* `Float32Array.from` yields one NaN per character rather than an error.
|
|
3658
|
+
*/
|
|
3637
3659
|
vector: number[];
|
|
3638
3660
|
/** Character offset where this chunk starts in the original message */
|
|
3639
3661
|
startOffset: number;
|
|
@@ -7373,6 +7395,17 @@ declare function embedAllMessages(ctx: StorageOperationsContext, options: Embedd
|
|
|
7373
7395
|
roles?: ("user" | "assistant")[];
|
|
7374
7396
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7375
7397
|
minContentLength?: number;
|
|
7398
|
+
/**
|
|
7399
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7400
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7401
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7402
|
+
* only — `tool_result` rows stay excluded.
|
|
7403
|
+
*
|
|
7404
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7405
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7406
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7407
|
+
*/
|
|
7408
|
+
reembedDiscarded?: boolean;
|
|
7376
7409
|
}): Promise<number>;
|
|
7377
7410
|
/**
|
|
7378
7411
|
* Chunk and embed a single message, storing chunk embeddings in the database.
|
|
@@ -7418,6 +7451,21 @@ declare function chunkAndEmbedAllMessages(ctx: StorageOperationsContext, options
|
|
|
7418
7451
|
rechunkExisting?: boolean;
|
|
7419
7452
|
/** Minimum content length to embed (default: 30). Shorter messages are skipped. */
|
|
7420
7453
|
minContentLength?: number;
|
|
7454
|
+
/**
|
|
7455
|
+
* Re-index rows the ciphertext sweep marked {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7456
|
+
* Off by default: this spends the user's own embedding credits, so it belongs
|
|
7457
|
+
* to an explicit user action, never to a background pass. Opens that marker
|
|
7458
|
+
* only — `tool_result` rows stay excluded.
|
|
7459
|
+
*
|
|
7460
|
+
* `rechunkExisting` cannot substitute for it: a discarded row has neither
|
|
7461
|
+
* chunks nor vector, so it never reaches that check and falls straight to the
|
|
7462
|
+
* origin gate.
|
|
7463
|
+
*
|
|
7464
|
+
* Per call, not a state change: a row that re-indexes successfully keeps its
|
|
7465
|
+
* marker, so a later pass that omits this flag skips it again. See
|
|
7466
|
+
* {@link CHUNKS_DISCARDED_ORIGIN}.
|
|
7467
|
+
*/
|
|
7468
|
+
reembedDiscarded?: boolean;
|
|
7421
7469
|
}): Promise<number>;
|
|
7422
7470
|
|
|
7423
7471
|
/**
|