@anuma/sdk 9.1.10 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/index.cjs +572 -239
- package/dist/expo/index.d.mts +86 -1
- package/dist/expo/index.d.ts +86 -1
- package/dist/expo/index.mjs +598 -265
- package/dist/index.cjs +444 -111
- package/dist/index.d.mts +86 -1
- package/dist/index.d.ts +86 -1
- package/dist/index.mjs +444 -111
- package/dist/react/index.cjs +619 -286
- package/dist/react/index.d.mts +86 -1
- package/dist/react/index.d.ts +86 -1
- package/dist/react/index.mjs +645 -312
- package/package.json +4 -2
package/dist/index.cjs
CHANGED
|
@@ -886,8 +886,8 @@ var createClient = (config = {}) => {
|
|
|
886
886
|
data = await response[parseAs]();
|
|
887
887
|
break;
|
|
888
888
|
case "json": {
|
|
889
|
-
const
|
|
890
|
-
data =
|
|
889
|
+
const text3 = await response.text();
|
|
890
|
+
data = text3 ? JSON.parse(text3) : {};
|
|
891
891
|
break;
|
|
892
892
|
}
|
|
893
893
|
case "stream":
|
|
@@ -1328,8 +1328,8 @@ var ATTACHED_FILES_OPEN_TAG = "<attached_files>";
|
|
|
1328
1328
|
var ATTACHED_FILES_CLOSE_TAG = "</attached_files>";
|
|
1329
1329
|
var ATTACHED_FILES_PREFIX = `${ATTACHED_FILES_OPEN_TAG}
|
|
1330
1330
|
The user attached the following file(s) to this message. Their extracted contents are below; when the user refers to "the attached file", "this file" or "the document", they mean these.`;
|
|
1331
|
-
function isAttachedFilesText(
|
|
1332
|
-
return typeof
|
|
1331
|
+
function isAttachedFilesText(text3) {
|
|
1332
|
+
return typeof text3 === "string" && text3.startsWith(ATTACHED_FILES_PREFIX) && text3.endsWith(ATTACHED_FILES_CLOSE_TAG);
|
|
1333
1333
|
}
|
|
1334
1334
|
function buildAttachedFilesText(fileContext) {
|
|
1335
1335
|
return `${ATTACHED_FILES_PREFIX}
|
|
@@ -1942,6 +1942,7 @@ function createNearbyRequest(options, apiPath = "/api/private-memories") {
|
|
|
1942
1942
|
}
|
|
1943
1943
|
throw error;
|
|
1944
1944
|
}
|
|
1945
|
+
if (response.status === 204) return null;
|
|
1945
1946
|
return await response.json();
|
|
1946
1947
|
} catch (error) {
|
|
1947
1948
|
if (timedOut) throw new RemoteMemoryError("Nearby request timed out", 408, "timeout");
|
|
@@ -1969,6 +1970,16 @@ var ChatStoreError = class extends Error {
|
|
|
1969
1970
|
// src/lib/chatStore/remote.ts
|
|
1970
1971
|
var kinds = /* @__PURE__ */ new Set(["conversation", "message", "summary", "project", "memory_links"]);
|
|
1971
1972
|
var cipherPattern = /^enc:v\d+:(?:[0-9a-f]{2}){28,}$/i;
|
|
1973
|
+
var maxPayload = 2 * 1024 * 1024;
|
|
1974
|
+
var storedDims = 1536;
|
|
1975
|
+
var maxRequestCiphertext = 8 * 1024 * 1024;
|
|
1976
|
+
var maxGetRefs = 50;
|
|
1977
|
+
var maxImportRecords = 100;
|
|
1978
|
+
var maxVerifyRevisions = 1e3;
|
|
1979
|
+
var maxChunksPerRecord = 16;
|
|
1980
|
+
var maxChunksPerRequest = 200;
|
|
1981
|
+
var maxAttachmentBytes = 50 * 1024 * 1024;
|
|
1982
|
+
var attachmentIdPattern = /^[0-9a-f]{32}$/;
|
|
1972
1983
|
var metadata = [
|
|
1973
1984
|
"kind",
|
|
1974
1985
|
"id",
|
|
@@ -2011,7 +2022,62 @@ function validRecord(value) {
|
|
|
2011
2022
|
))
|
|
2012
2023
|
return false;
|
|
2013
2024
|
const root = value.kind === "conversation" || value.kind === "project";
|
|
2014
|
-
return root ? value.conversationId === ""
|
|
2025
|
+
return root ? value.conversationId === "" : id(value.conversationId);
|
|
2026
|
+
}
|
|
2027
|
+
function embeddings(write) {
|
|
2028
|
+
if (write.embeddings === void 0) return true;
|
|
2029
|
+
return Array.isArray(write.embeddings) && !write.record.isDeleted && (write.record.kind === "message" || write.record.kind === "summary") && write.embeddings.length <= maxChunksPerRecord && (write.embeddingModel === void 0 || typeof write.embeddingModel === "string" && write.embeddingModel.length <= 256) && Array.from({ length: write.embeddings.length }, (_, i) => i).every(
|
|
2030
|
+
(i) => vector(write.embeddings?.[i])
|
|
2031
|
+
);
|
|
2032
|
+
}
|
|
2033
|
+
function vector(v) {
|
|
2034
|
+
if (!Array.isArray(v) || v.length !== 1536 && v.length !== 4096) return false;
|
|
2035
|
+
let nonZero = false;
|
|
2036
|
+
for (let i = 0; i < v.length; i++) {
|
|
2037
|
+
const x = v[i];
|
|
2038
|
+
if (typeof x !== "number" || !Number.isFinite(x)) return false;
|
|
2039
|
+
if (i < storedDims && x !== 0) nonZero = true;
|
|
2040
|
+
}
|
|
2041
|
+
return nonZero;
|
|
2042
|
+
}
|
|
2043
|
+
function vectorFields(write) {
|
|
2044
|
+
return {
|
|
2045
|
+
...write.embeddings !== void 0 ? { embeddings: write.embeddings.map((v) => v.slice(0, storedDims)) } : {},
|
|
2046
|
+
...write.embeddingModel !== void 0 ? { embedding_model: write.embeddingModel } : {}
|
|
2047
|
+
};
|
|
2048
|
+
}
|
|
2049
|
+
function text2(value) {
|
|
2050
|
+
return typeof value === "string";
|
|
2051
|
+
}
|
|
2052
|
+
function timestamp(value) {
|
|
2053
|
+
return typeof value === "string" && Number.isFinite(Date.parse(value));
|
|
2054
|
+
}
|
|
2055
|
+
function presigned(value) {
|
|
2056
|
+
if (!object(value) || !text2(value.url) || !/^https:\/\//.test(value.url) || !text2(value.method) || !object(value.headers) || !Object.values(value.headers).every(text2) || !timestamp(value.expires_at))
|
|
2057
|
+
invalid();
|
|
2058
|
+
return {
|
|
2059
|
+
url: value.url,
|
|
2060
|
+
method: value.method,
|
|
2061
|
+
headers: value.headers,
|
|
2062
|
+
expiresAt: value.expires_at
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
function attachment(value) {
|
|
2066
|
+
if (!object(value) || !text2(value.attachment_id) || !attachmentIdPattern.test(value.attachment_id) || !integer(value.size_bytes, 1) || value.state !== "pending" && value.state !== "committed" || value.owner_kind !== void 0 && value.owner_kind !== "message" && value.owner_kind !== "summary" || value.owner_id !== void 0 && !id(value.owner_id) || !timestamp(value.created_at) || value.committed_at !== void 0 && !timestamp(value.committed_at))
|
|
2067
|
+
invalid();
|
|
2068
|
+
return {
|
|
2069
|
+
id: value.attachment_id,
|
|
2070
|
+
sizeBytes: value.size_bytes,
|
|
2071
|
+
state: value.state,
|
|
2072
|
+
...value.owner_kind !== void 0 ? { ownerKind: value.owner_kind } : {},
|
|
2073
|
+
...value.owner_id !== void 0 ? { ownerId: value.owner_id } : {},
|
|
2074
|
+
createdAt: value.created_at,
|
|
2075
|
+
...value.committed_at !== void 0 ? { committedAt: value.committed_at } : {}
|
|
2076
|
+
};
|
|
2077
|
+
}
|
|
2078
|
+
function ref(value) {
|
|
2079
|
+
if (!object(value) || !text2(value.kind) || !kinds.has(value.kind) || !id(value.id)) invalid();
|
|
2080
|
+
return { kind: value.kind, id: value.id };
|
|
2015
2081
|
}
|
|
2016
2082
|
function sameMetadata(a, b) {
|
|
2017
2083
|
return metadata.every((key) => a[key] === b[key]);
|
|
@@ -2067,8 +2133,11 @@ async function createRemoteChatStore(options) {
|
|
|
2067
2133
|
isDeleted: r.is_deleted,
|
|
2068
2134
|
data: {}
|
|
2069
2135
|
};
|
|
2070
|
-
if (!validRecord(plain) || typeof r.payload !== "string" || r.payload.length >
|
|
2136
|
+
if (!validRecord(plain) || typeof r.payload !== "string" || r.payload.length > maxPayload)
|
|
2071
2137
|
invalid();
|
|
2138
|
+
if (plain.isDeleted && r.payload === "")
|
|
2139
|
+
return { record: plain, version: value.version, serverUpdatedAt: value.server_updated_at };
|
|
2140
|
+
if (!cipherPattern.test(r.payload)) invalid();
|
|
2072
2141
|
const envelope = JSON.parse(await decrypt(r.payload));
|
|
2073
2142
|
if (!object(envelope) || envelope.schema !== 1 || envelope.account_id !== accountId || envelope.key_id !== keyId || !validRecord(envelope.record) || !sameMetadata(plain, envelope.record))
|
|
2074
2143
|
invalid();
|
|
@@ -2078,30 +2147,46 @@ async function createRemoteChatStore(options) {
|
|
|
2078
2147
|
serverUpdatedAt: value.server_updated_at
|
|
2079
2148
|
};
|
|
2080
2149
|
};
|
|
2081
|
-
const
|
|
2082
|
-
if (
|
|
2150
|
+
const seal = async (record2, signal) => {
|
|
2151
|
+
if (record2.isDeleted) return "";
|
|
2152
|
+
const payload = await encrypt(
|
|
2153
|
+
JSON.stringify({ schema: 1, account_id: accountId, key_id: keyId, record: record2 })
|
|
2154
|
+
);
|
|
2155
|
+
signal?.throwIfAborted();
|
|
2156
|
+
if (!cipherPattern.test(payload) || payload.length > maxPayload) invalid();
|
|
2157
|
+
return payload;
|
|
2158
|
+
};
|
|
2159
|
+
const capture = (items, max, signal) => {
|
|
2160
|
+
if (!Array.isArray(items) || items.length < 1 || items.length > max) invalid();
|
|
2083
2161
|
const seen = /* @__PURE__ */ new Set();
|
|
2084
|
-
|
|
2162
|
+
let chunks = 0;
|
|
2163
|
+
return items.map((item) => {
|
|
2085
2164
|
signal?.throwIfAborted();
|
|
2086
|
-
if (!validRecord(
|
|
2087
|
-
|
|
2088
|
-
|
|
2165
|
+
if (!validRecord(item.record) || !embeddings(item)) invalid();
|
|
2166
|
+
chunks += item.embeddings?.length ?? 0;
|
|
2167
|
+
if (chunks > maxChunksPerRequest) invalid();
|
|
2168
|
+
const copy = JSON.parse(JSON.stringify(item));
|
|
2169
|
+
const recordKey = `${copy.record.kind}/${copy.record.id}`;
|
|
2089
2170
|
if (seen.has(recordKey)) invalid();
|
|
2090
2171
|
seen.add(recordKey);
|
|
2091
|
-
return
|
|
2172
|
+
return copy;
|
|
2092
2173
|
});
|
|
2174
|
+
};
|
|
2175
|
+
const putMany = async (writes, signal) => {
|
|
2176
|
+
if (Array.isArray(writes) && writes.some((w) => !integer(w.expectedVersion)))
|
|
2177
|
+
invalid();
|
|
2178
|
+
const captured = capture(writes, 25, signal);
|
|
2093
2179
|
let bytes = 0;
|
|
2094
2180
|
const prepared = [];
|
|
2095
2181
|
for (const write of captured) {
|
|
2096
|
-
const
|
|
2097
|
-
const payload = await encrypt(
|
|
2098
|
-
JSON.stringify({ schema: 1, account_id: accountId, key_id: keyId, record: record2 })
|
|
2099
|
-
);
|
|
2100
|
-
signal?.throwIfAborted();
|
|
2101
|
-
if (!cipherPattern.test(payload) || payload.length > 2 * 1024 * 1024) invalid();
|
|
2182
|
+
const payload = await seal(write.record, signal);
|
|
2102
2183
|
bytes += payload.length;
|
|
2103
|
-
if (bytes >
|
|
2104
|
-
prepared.push({
|
|
2184
|
+
if (bytes > maxRequestCiphertext) invalid();
|
|
2185
|
+
prepared.push({
|
|
2186
|
+
record: wire(write.record, payload),
|
|
2187
|
+
expected_version: write.expectedVersion,
|
|
2188
|
+
...vectorFields(write)
|
|
2189
|
+
});
|
|
2105
2190
|
}
|
|
2106
2191
|
const result = await request(
|
|
2107
2192
|
"/records/batch",
|
|
@@ -2116,7 +2201,8 @@ async function createRemoteChatStore(options) {
|
|
|
2116
2201
|
const out = [];
|
|
2117
2202
|
for (let i = 0; i < result.items.length; i++) {
|
|
2118
2203
|
const value = result.items[i];
|
|
2119
|
-
|
|
2204
|
+
const deleted = prepared[i].record.is_deleted;
|
|
2205
|
+
if (!object(value) || !object(value.record) || value.record.kind !== prepared[i].record.kind || value.record.id !== prepared[i].record.id || value.record.payload !== prepared[i].record.payload || (deleted ? value.record.is_deleted !== true || !integer(value.version, 1) : value.version !== prepared[i].expected_version + 1))
|
|
2120
2206
|
invalid();
|
|
2121
2207
|
out.push(await decode(value));
|
|
2122
2208
|
}
|
|
@@ -2145,7 +2231,7 @@ async function createRemoteChatStore(options) {
|
|
|
2145
2231
|
async list(query) {
|
|
2146
2232
|
const limit = query.limit ?? 20;
|
|
2147
2233
|
const parent = query.conversationId ?? "";
|
|
2148
|
-
if (!kinds.has(query.kind) || !integer(limit, 1) || query.includeDeleted !== void 0 && typeof query.includeDeleted !== "boolean" || limit > 50 || (query.cursor?.length ?? 0) > 2048 || (query.kind === "conversation" || query.kind === "project" ? parent !== "" : !id(parent)))
|
|
2234
|
+
if (!kinds.has(query.kind) || !integer(limit, 1) || query.includeDeleted !== void 0 && typeof query.includeDeleted !== "boolean" || limit > (query.kind === "message" ? 200 : 50) || (query.cursor?.length ?? 0) > 2048 || (query.kind === "conversation" || query.kind === "project" ? parent !== "" : !id(parent)))
|
|
2149
2235
|
invalid();
|
|
2150
2236
|
const params = new URLSearchParams({
|
|
2151
2237
|
account_id: String(accountId),
|
|
@@ -2156,7 +2242,7 @@ async function createRemoteChatStore(options) {
|
|
|
2156
2242
|
});
|
|
2157
2243
|
if (query.cursor) params.set("cursor", query.cursor);
|
|
2158
2244
|
const value = await request(`/records?${params}`, {}, query.signal);
|
|
2159
|
-
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit || value.next_cursor !== void 0 && (typeof value.next_cursor !== "string" || value.next_cursor.length > 2048))
|
|
2245
|
+
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit || value.next_cursor !== void 0 && (typeof value.next_cursor !== "string" || value.next_cursor.length > 2048) || value.total !== void 0 && !integer(value.total))
|
|
2160
2246
|
invalid();
|
|
2161
2247
|
const items = [];
|
|
2162
2248
|
for (const item of value.items) {
|
|
@@ -2166,11 +2252,258 @@ async function createRemoteChatStore(options) {
|
|
|
2166
2252
|
items.push(result);
|
|
2167
2253
|
}
|
|
2168
2254
|
query.signal?.throwIfAborted();
|
|
2169
|
-
return {
|
|
2255
|
+
return {
|
|
2256
|
+
items,
|
|
2257
|
+
...value.next_cursor ? { nextCursor: value.next_cursor } : {},
|
|
2258
|
+
...value.total !== void 0 ? { total: value.total } : {}
|
|
2259
|
+
};
|
|
2260
|
+
},
|
|
2261
|
+
putMany,
|
|
2262
|
+
async getMany(refs, signal) {
|
|
2263
|
+
if (!Array.isArray(refs)) invalid();
|
|
2264
|
+
const wanted = /* @__PURE__ */ new Map();
|
|
2265
|
+
for (const r of refs) {
|
|
2266
|
+
const checked = ref(r);
|
|
2267
|
+
wanted.set(`${checked.kind}/${checked.id}`, checked);
|
|
2268
|
+
}
|
|
2269
|
+
const found = /* @__PURE__ */ new Map();
|
|
2270
|
+
let pending = [...wanted.values()];
|
|
2271
|
+
while (pending.length > 0) {
|
|
2272
|
+
const batch = pending.slice(0, maxGetRefs);
|
|
2273
|
+
pending = pending.slice(maxGetRefs);
|
|
2274
|
+
const value = await request(
|
|
2275
|
+
"/records/get",
|
|
2276
|
+
{ method: "POST", body: JSON.stringify({ account_id: accountId, refs: batch }) },
|
|
2277
|
+
signal
|
|
2278
|
+
);
|
|
2279
|
+
if (!object(value) || !Array.isArray(value.items) || value.remaining !== void 0 && !Array.isArray(value.remaining))
|
|
2280
|
+
invalid();
|
|
2281
|
+
const asked = new Set(batch.map((r) => `${r.kind}/${r.id}`));
|
|
2282
|
+
for (const item of value.items) {
|
|
2283
|
+
const snapshot = await decode(item);
|
|
2284
|
+
const key = `${snapshot.record.kind}/${snapshot.record.id}`;
|
|
2285
|
+
if (!asked.has(key) || found.has(key)) invalid();
|
|
2286
|
+
found.set(key, snapshot);
|
|
2287
|
+
}
|
|
2288
|
+
const remaining = (value.remaining ?? []).map(ref);
|
|
2289
|
+
if (remaining.some((r) => !asked.has(`${r.kind}/${r.id}`) || found.has(`${r.kind}/${r.id}`)))
|
|
2290
|
+
invalid();
|
|
2291
|
+
if (remaining.length > 0 && value.items.length === 0) invalid();
|
|
2292
|
+
pending = [...remaining, ...pending];
|
|
2293
|
+
}
|
|
2294
|
+
signal?.throwIfAborted();
|
|
2295
|
+
return [...wanted.keys()].flatMap((key) => {
|
|
2296
|
+
const snapshot = found.get(key);
|
|
2297
|
+
return snapshot ? [snapshot] : [];
|
|
2298
|
+
});
|
|
2299
|
+
},
|
|
2300
|
+
async account(signal) {
|
|
2301
|
+
const value = await request("/account", {}, signal);
|
|
2302
|
+
if (!object(value) || value.account_id !== accountId || !text2(value.key_id) || value.state !== "none" && value.state !== "migrating" && value.state !== "active" || !integer(value.record_count) || !integer(value.storage_bytes) || !integer(value.vector_count))
|
|
2303
|
+
invalid();
|
|
2304
|
+
if (value.key_id && value.key_id !== keyId)
|
|
2305
|
+
throw new ChatStoreError("Canonical chat key does not match", 409, "key_mismatch");
|
|
2306
|
+
const out = {
|
|
2307
|
+
keyId: value.key_id,
|
|
2308
|
+
state: value.state,
|
|
2309
|
+
recordCount: value.record_count,
|
|
2310
|
+
storageBytes: value.storage_bytes,
|
|
2311
|
+
vectorCount: value.vector_count
|
|
2312
|
+
};
|
|
2313
|
+
return out;
|
|
2314
|
+
},
|
|
2315
|
+
async importRecords(records, signal) {
|
|
2316
|
+
const captured = capture(records, maxImportRecords, signal);
|
|
2317
|
+
let bytes = 0;
|
|
2318
|
+
const prepared = [];
|
|
2319
|
+
for (const item of captured) {
|
|
2320
|
+
const payload = await seal(item.record, signal);
|
|
2321
|
+
bytes += payload.length;
|
|
2322
|
+
if (bytes > maxRequestCiphertext) invalid();
|
|
2323
|
+
prepared.push({ record: wire(item.record, payload), ...vectorFields(item) });
|
|
2324
|
+
}
|
|
2325
|
+
const value = await request(
|
|
2326
|
+
"/import",
|
|
2327
|
+
{
|
|
2328
|
+
method: "POST",
|
|
2329
|
+
body: JSON.stringify({ account_id: accountId, key_id: keyId, records: prepared })
|
|
2330
|
+
},
|
|
2331
|
+
signal
|
|
2332
|
+
);
|
|
2333
|
+
if (!object(value) || !integer(value.written) || value.written > prepared.length) invalid();
|
|
2334
|
+
return value.written;
|
|
2335
|
+
},
|
|
2336
|
+
async verify(revisions, signal) {
|
|
2337
|
+
if (!Array.isArray(revisions) || revisions.length > maxVerifyRevisions) invalid();
|
|
2338
|
+
const asked = /* @__PURE__ */ new Set();
|
|
2339
|
+
const wireRevisions = revisions.map((r) => {
|
|
2340
|
+
const checked = ref(r);
|
|
2341
|
+
if (!integer(r.updatedAt, 1) || typeof r.isDeleted !== "boolean") invalid();
|
|
2342
|
+
asked.add(`${checked.kind}/${checked.id}`);
|
|
2343
|
+
return {
|
|
2344
|
+
kind: checked.kind,
|
|
2345
|
+
id: checked.id,
|
|
2346
|
+
updated_at: r.updatedAt,
|
|
2347
|
+
is_deleted: r.isDeleted
|
|
2348
|
+
};
|
|
2349
|
+
});
|
|
2350
|
+
if (wireRevisions.length === 0) return [];
|
|
2351
|
+
const value = await request(
|
|
2352
|
+
"/verify",
|
|
2353
|
+
{
|
|
2354
|
+
method: "POST",
|
|
2355
|
+
body: JSON.stringify({ account_id: accountId, revisions: wireRevisions })
|
|
2356
|
+
},
|
|
2357
|
+
signal
|
|
2358
|
+
);
|
|
2359
|
+
if (!object(value) || !Array.isArray(value.missing)) invalid();
|
|
2360
|
+
const missing = value.missing.map(ref);
|
|
2361
|
+
if (missing.some((r) => !asked.has(`${r.kind}/${r.id}`))) invalid();
|
|
2362
|
+
return missing;
|
|
2363
|
+
},
|
|
2364
|
+
async activate(signal) {
|
|
2365
|
+
const value = await request(
|
|
2366
|
+
"/account",
|
|
2367
|
+
{ method: "PUT", body: JSON.stringify({ account_id: accountId, state: "active" }) },
|
|
2368
|
+
signal
|
|
2369
|
+
);
|
|
2370
|
+
if (!object(value) || value.state !== "active" || typeof value.transitioned !== "boolean")
|
|
2371
|
+
invalid();
|
|
2372
|
+
return value.transitioned;
|
|
2373
|
+
},
|
|
2374
|
+
async changes(cursor, options2) {
|
|
2375
|
+
const limit = options2?.limit ?? 100;
|
|
2376
|
+
if (!integer(limit, 1) || limit > 200 || cursor !== void 0 && (!text2(cursor) || cursor.length > 2048))
|
|
2377
|
+
invalid();
|
|
2378
|
+
const params = new URLSearchParams({ account_id: String(accountId), limit: String(limit) });
|
|
2379
|
+
if (cursor) params.set("cursor", cursor);
|
|
2380
|
+
const value = await request(`/changes?${params}`, {}, options2?.signal);
|
|
2381
|
+
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit || value.next_cursor !== void 0 && !text2(value.next_cursor) || value.changes_cursor !== void 0 && !text2(value.changes_cursor))
|
|
2382
|
+
invalid();
|
|
2383
|
+
const items = [];
|
|
2384
|
+
for (const item of value.items) items.push(await decode(item));
|
|
2385
|
+
options2?.signal?.throwIfAborted();
|
|
2386
|
+
return {
|
|
2387
|
+
items,
|
|
2388
|
+
...value.next_cursor ? { nextCursor: value.next_cursor } : {},
|
|
2389
|
+
changesCursor: value.changes_cursor ?? cursor ?? ""
|
|
2390
|
+
};
|
|
2391
|
+
},
|
|
2392
|
+
async search(embedding, options2) {
|
|
2393
|
+
const limit = options2?.limit ?? 20;
|
|
2394
|
+
const kindsFilter = options2?.kinds;
|
|
2395
|
+
if (!Array.isArray(embedding) || !embeddings({
|
|
2396
|
+
record: { ...emptyMessage, isDeleted: false },
|
|
2397
|
+
embeddings: [embedding]
|
|
2398
|
+
}) || !integer(limit, 1) || limit > 50 || kindsFilter !== void 0 && (!Array.isArray(kindsFilter) || kindsFilter.length === 0 || kindsFilter.some((k) => k !== "message" && k !== "summary")) || options2?.conversationId !== void 0 && !id(options2.conversationId))
|
|
2399
|
+
invalid();
|
|
2400
|
+
const value = await request(
|
|
2401
|
+
"/search",
|
|
2402
|
+
{
|
|
2403
|
+
method: "POST",
|
|
2404
|
+
body: JSON.stringify({
|
|
2405
|
+
account_id: accountId,
|
|
2406
|
+
embedding: embedding.slice(0, storedDims),
|
|
2407
|
+
limit,
|
|
2408
|
+
...kindsFilter ? { kinds: kindsFilter } : {},
|
|
2409
|
+
...options2?.conversationId ? { conversation_id: options2.conversationId } : {}
|
|
2410
|
+
})
|
|
2411
|
+
},
|
|
2412
|
+
options2?.signal
|
|
2413
|
+
);
|
|
2414
|
+
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit) invalid();
|
|
2415
|
+
const hits = [];
|
|
2416
|
+
for (const item of value.items) {
|
|
2417
|
+
if (!object(item) || !integer(item.chunk) || typeof item.score !== "number" || !Number.isFinite(item.score))
|
|
2418
|
+
invalid();
|
|
2419
|
+
const snapshot = await decode(item);
|
|
2420
|
+
if (snapshot.record.isDeleted || kindsFilter !== void 0 && !kindsFilter.includes(snapshot.record.kind) || snapshot.record.kind !== "message" && snapshot.record.kind !== "summary" || options2?.conversationId !== void 0 && snapshot.record.conversationId !== options2.conversationId)
|
|
2421
|
+
invalid();
|
|
2422
|
+
hits.push({ ...snapshot, chunk: item.chunk, score: item.score });
|
|
2423
|
+
}
|
|
2424
|
+
options2?.signal?.throwIfAborted();
|
|
2425
|
+
return hits;
|
|
2426
|
+
},
|
|
2427
|
+
async deleteAll(signal) {
|
|
2428
|
+
const value = await request(`/records?account_id=${accountId}`, { method: "DELETE" }, signal);
|
|
2429
|
+
if (!object(value) || !integer(value.deleted)) invalid();
|
|
2430
|
+
return value.deleted;
|
|
2170
2431
|
},
|
|
2171
|
-
|
|
2432
|
+
attachments: {
|
|
2433
|
+
async create(sizeBytes, signal) {
|
|
2434
|
+
if (!integer(sizeBytes, 1) || sizeBytes > maxAttachmentBytes) invalid();
|
|
2435
|
+
const value = await request(
|
|
2436
|
+
"/attachments",
|
|
2437
|
+
{
|
|
2438
|
+
method: "POST",
|
|
2439
|
+
body: JSON.stringify({ account_id: accountId, size_bytes: sizeBytes })
|
|
2440
|
+
},
|
|
2441
|
+
signal
|
|
2442
|
+
);
|
|
2443
|
+
if (!object(value)) invalid();
|
|
2444
|
+
const created = attachment(value.attachment);
|
|
2445
|
+
if (created.sizeBytes !== sizeBytes || created.state !== "pending") invalid();
|
|
2446
|
+
return { attachment: created, upload: presigned(value.upload) };
|
|
2447
|
+
},
|
|
2448
|
+
async commit(attachmentId, owner, signal) {
|
|
2449
|
+
if (!attachmentIdPattern.test(attachmentId) || !object(owner) || owner.kind !== "message" && owner.kind !== "summary" || !id(owner.id))
|
|
2450
|
+
invalid();
|
|
2451
|
+
const value = await request(
|
|
2452
|
+
`/attachments/${attachmentId}/commit`,
|
|
2453
|
+
{
|
|
2454
|
+
method: "POST",
|
|
2455
|
+
body: JSON.stringify({
|
|
2456
|
+
account_id: accountId,
|
|
2457
|
+
owner_kind: owner.kind,
|
|
2458
|
+
owner_id: owner.id
|
|
2459
|
+
})
|
|
2460
|
+
},
|
|
2461
|
+
signal
|
|
2462
|
+
);
|
|
2463
|
+
if (!object(value)) invalid();
|
|
2464
|
+
const committed = attachment(value.attachment);
|
|
2465
|
+
if (committed.id !== attachmentId || committed.state !== "committed" || committed.ownerKind !== owner.kind || committed.ownerId !== owner.id)
|
|
2466
|
+
invalid();
|
|
2467
|
+
return committed;
|
|
2468
|
+
},
|
|
2469
|
+
async get(attachmentId, signal) {
|
|
2470
|
+
if (!attachmentIdPattern.test(attachmentId)) invalid();
|
|
2471
|
+
let value;
|
|
2472
|
+
try {
|
|
2473
|
+
value = await request(`/attachments/${attachmentId}?account_id=${accountId}`, {}, signal);
|
|
2474
|
+
} catch (error) {
|
|
2475
|
+
if (error instanceof ChatStoreError && error.status === 404) return null;
|
|
2476
|
+
throw error;
|
|
2477
|
+
}
|
|
2478
|
+
if (!object(value)) invalid();
|
|
2479
|
+
const found = attachment(value.attachment);
|
|
2480
|
+
if (found.id !== attachmentId) invalid();
|
|
2481
|
+
return {
|
|
2482
|
+
attachment: found,
|
|
2483
|
+
...value.download !== void 0 ? { download: presigned(value.download) } : {}
|
|
2484
|
+
};
|
|
2485
|
+
},
|
|
2486
|
+
async delete(attachmentId, signal) {
|
|
2487
|
+
if (!attachmentIdPattern.test(attachmentId)) invalid();
|
|
2488
|
+
await request(
|
|
2489
|
+
`/attachments/${attachmentId}?account_id=${accountId}`,
|
|
2490
|
+
{ method: "DELETE" },
|
|
2491
|
+
signal
|
|
2492
|
+
);
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2172
2495
|
};
|
|
2173
2496
|
}
|
|
2497
|
+
var emptyMessage = {
|
|
2498
|
+
kind: "message",
|
|
2499
|
+
id: "query",
|
|
2500
|
+
conversationId: "query",
|
|
2501
|
+
sortKey: 0,
|
|
2502
|
+
createdAt: 1,
|
|
2503
|
+
updatedAt: 1,
|
|
2504
|
+
isDeleted: false,
|
|
2505
|
+
data: {}
|
|
2506
|
+
};
|
|
2174
2507
|
|
|
2175
2508
|
// src/lib/db/chat/lazyDecrypt.ts
|
|
2176
2509
|
var LRU_CAPACITY = 256;
|
|
@@ -2284,19 +2617,19 @@ function base64ToUint8Array(base64) {
|
|
|
2284
2617
|
// src/lib/memoryEngine/vectorEncoding.ts
|
|
2285
2618
|
var BYTES_PER_FLOAT32 = 4;
|
|
2286
2619
|
var CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
|
|
2287
|
-
function encodeChunkVector(
|
|
2288
|
-
const f32 =
|
|
2620
|
+
function encodeChunkVector(vector2) {
|
|
2621
|
+
const f32 = vector2 instanceof Float32Array ? vector2 : Float32Array.from(vector2);
|
|
2289
2622
|
return uint8ArrayToBase64(new Uint8Array(f32.buffer, f32.byteOffset, f32.byteLength));
|
|
2290
2623
|
}
|
|
2291
|
-
function decodeChunkVector(
|
|
2292
|
-
if (!
|
|
2293
|
-
if (typeof
|
|
2294
|
-
if (
|
|
2624
|
+
function decodeChunkVector(vector2, onMalformed) {
|
|
2625
|
+
if (!vector2 || vector2.length === 0) return new Float32Array(0);
|
|
2626
|
+
if (typeof vector2 !== "string") return Float32Array.from(vector2);
|
|
2627
|
+
if (vector2.length % 4 !== 0 || !CANONICAL_BASE64.test(vector2)) {
|
|
2295
2628
|
onMalformed?.();
|
|
2296
2629
|
return new Float32Array(0);
|
|
2297
2630
|
}
|
|
2298
2631
|
try {
|
|
2299
|
-
const bytes = base64ToUint8Array(
|
|
2632
|
+
const bytes = base64ToUint8Array(vector2);
|
|
2300
2633
|
if (bytes.byteLength === 0 || bytes.byteLength % BYTES_PER_FLOAT32 !== 0) {
|
|
2301
2634
|
onMalformed?.();
|
|
2302
2635
|
return new Float32Array(0);
|
|
@@ -2312,49 +2645,49 @@ function decodeChunkVector(vector, onMalformed) {
|
|
|
2312
2645
|
var DEFAULT_CHUNK_SIZE = 400;
|
|
2313
2646
|
var DEFAULT_CHUNK_OVERLAP = 50;
|
|
2314
2647
|
var DEFAULT_MIN_CHUNK_SIZE = 50;
|
|
2315
|
-
function splitIntoSentences(
|
|
2648
|
+
function splitIntoSentences(text3) {
|
|
2316
2649
|
const sentenceRegex = /[.!?]+[\s"')}\]]*(?=\s|$)/g;
|
|
2317
2650
|
const sentences = [];
|
|
2318
2651
|
let lastIndex = 0;
|
|
2319
2652
|
let match;
|
|
2320
|
-
while ((match = sentenceRegex.exec(
|
|
2653
|
+
while ((match = sentenceRegex.exec(text3)) !== null) {
|
|
2321
2654
|
const endIndex = match.index + match[0].length;
|
|
2322
|
-
const sentence =
|
|
2655
|
+
const sentence = text3.slice(lastIndex, endIndex).trim();
|
|
2323
2656
|
if (sentence) {
|
|
2324
2657
|
sentences.push(sentence);
|
|
2325
2658
|
}
|
|
2326
2659
|
lastIndex = endIndex;
|
|
2327
2660
|
}
|
|
2328
|
-
const remaining =
|
|
2661
|
+
const remaining = text3.slice(lastIndex).trim();
|
|
2329
2662
|
if (remaining) {
|
|
2330
2663
|
sentences.push(remaining);
|
|
2331
2664
|
}
|
|
2332
2665
|
return sentences;
|
|
2333
2666
|
}
|
|
2334
|
-
function chunkText(
|
|
2667
|
+
function chunkText(text3, options) {
|
|
2335
2668
|
const {
|
|
2336
2669
|
chunkSize = DEFAULT_CHUNK_SIZE,
|
|
2337
2670
|
chunkOverlap: requestedOverlap = DEFAULT_CHUNK_OVERLAP,
|
|
2338
2671
|
minChunkSize = DEFAULT_MIN_CHUNK_SIZE
|
|
2339
2672
|
} = options ?? {};
|
|
2340
2673
|
const chunkOverlap = Math.min(requestedOverlap, chunkSize - 1);
|
|
2341
|
-
if (
|
|
2674
|
+
if (text3.length <= chunkSize) {
|
|
2342
2675
|
return [
|
|
2343
2676
|
{
|
|
2344
|
-
text:
|
|
2677
|
+
text: text3.trim(),
|
|
2345
2678
|
startOffset: 0,
|
|
2346
|
-
endOffset:
|
|
2679
|
+
endOffset: text3.length
|
|
2347
2680
|
}
|
|
2348
2681
|
];
|
|
2349
2682
|
}
|
|
2350
|
-
const sentences = splitIntoSentences(
|
|
2683
|
+
const sentences = splitIntoSentences(text3);
|
|
2351
2684
|
const chunks = [];
|
|
2352
2685
|
let currentChunkSentences = [];
|
|
2353
2686
|
let currentChunkStart = 0;
|
|
2354
2687
|
let currentOffset = 0;
|
|
2355
2688
|
for (let i = 0; i < sentences.length; i++) {
|
|
2356
2689
|
const sentence = sentences[i];
|
|
2357
|
-
let sentenceStart =
|
|
2690
|
+
let sentenceStart = text3.indexOf(sentence, currentOffset);
|
|
2358
2691
|
if (sentenceStart === -1) {
|
|
2359
2692
|
sentenceStart = currentOffset;
|
|
2360
2693
|
}
|
|
@@ -2401,7 +2734,7 @@ function chunkText(text2, options) {
|
|
|
2401
2734
|
if (overlapSentences.length > 0) {
|
|
2402
2735
|
currentChunkSentences = [...overlapSentences, sentence];
|
|
2403
2736
|
const overlapText = overlapSentences[0];
|
|
2404
|
-
const overlapStart =
|
|
2737
|
+
const overlapStart = text3.lastIndexOf(overlapText, currentOffset);
|
|
2405
2738
|
currentChunkStart = overlapStart === -1 ? sentenceStart : overlapStart;
|
|
2406
2739
|
} else {
|
|
2407
2740
|
currentChunkSentences = [sentence];
|
|
@@ -2421,18 +2754,18 @@ function chunkText(text2, options) {
|
|
|
2421
2754
|
chunks.push({
|
|
2422
2755
|
text: chunkText2,
|
|
2423
2756
|
startOffset: currentChunkStart,
|
|
2424
|
-
endOffset:
|
|
2757
|
+
endOffset: text3.length
|
|
2425
2758
|
});
|
|
2426
2759
|
}
|
|
2427
2760
|
}
|
|
2428
2761
|
if (chunks.length === 0) {
|
|
2429
|
-
const trimmedText =
|
|
2762
|
+
const trimmedText = text3.trim();
|
|
2430
2763
|
if (trimmedText.length > 0) {
|
|
2431
2764
|
return [
|
|
2432
2765
|
{
|
|
2433
2766
|
text: trimmedText,
|
|
2434
2767
|
startOffset: 0,
|
|
2435
|
-
endOffset:
|
|
2768
|
+
endOffset: text3.length
|
|
2436
2769
|
}
|
|
2437
2770
|
];
|
|
2438
2771
|
}
|
|
@@ -2548,22 +2881,22 @@ function embeddingErrorFrom(response) {
|
|
|
2548
2881
|
const message = typeof response.error === "object" && response.error && "error" in response.error ? response.error.error : "API embedding failed";
|
|
2549
2882
|
return new EmbeddingHttpError(message, response.response?.status);
|
|
2550
2883
|
}
|
|
2551
|
-
async function generateEmbedding(
|
|
2884
|
+
async function generateEmbedding(text3, options) {
|
|
2552
2885
|
const total = options.totalTimeoutMs;
|
|
2553
|
-
if (total === void 0) return embedOne(
|
|
2554
|
-
return withDeadline((signal) => embedOne(
|
|
2886
|
+
if (total === void 0) return embedOne(text3, options, void 0);
|
|
2887
|
+
return withDeadline((signal) => embedOne(text3, options, signal), total, "embedding");
|
|
2555
2888
|
}
|
|
2556
2889
|
var { inFlightByCache, cachedEmbeddingModels } = getEmbeddingCacheState();
|
|
2557
|
-
async function embedOne(
|
|
2890
|
+
async function embedOne(text3, options, outer) {
|
|
2558
2891
|
const { cache } = options;
|
|
2559
|
-
if (!cache) return Array.from(await requestEmbedding(
|
|
2892
|
+
if (!cache) return Array.from(await requestEmbedding(text3, options, outer));
|
|
2560
2893
|
const model = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
2561
|
-
const cached = cache.get(
|
|
2894
|
+
const cached = cache.get(text3);
|
|
2562
2895
|
const cachedModel = cached && cachedEmbeddingModels.get(cached);
|
|
2563
2896
|
if (cached && (cachedModel === void 0 || cachedModel === model)) return Array.from(cached);
|
|
2564
2897
|
const inFlight = inFlightByCache.get(cache) ?? /* @__PURE__ */ new Map();
|
|
2565
2898
|
inFlightByCache.set(cache, inFlight);
|
|
2566
|
-
const requestKey = JSON.stringify([model,
|
|
2899
|
+
const requestKey = JSON.stringify([model, text3]);
|
|
2567
2900
|
const pending = inFlight.get(requestKey);
|
|
2568
2901
|
if (pending) {
|
|
2569
2902
|
try {
|
|
@@ -2572,18 +2905,18 @@ async function embedOne(text2, options, outer) {
|
|
|
2572
2905
|
if (outer?.aborted) throw err;
|
|
2573
2906
|
}
|
|
2574
2907
|
}
|
|
2575
|
-
const request = requestEmbedding(
|
|
2908
|
+
const request = requestEmbedding(text3, options, outer);
|
|
2576
2909
|
inFlight.set(requestKey, request);
|
|
2577
2910
|
try {
|
|
2578
2911
|
const embedding = await request;
|
|
2579
2912
|
cachedEmbeddingModels.set(embedding, model);
|
|
2580
|
-
cache.set(
|
|
2913
|
+
cache.set(text3, embedding);
|
|
2581
2914
|
return Array.from(embedding);
|
|
2582
2915
|
} finally {
|
|
2583
2916
|
if (inFlight.get(requestKey) === request) inFlight.delete(requestKey);
|
|
2584
2917
|
}
|
|
2585
2918
|
}
|
|
2586
|
-
async function requestEmbedding(
|
|
2919
|
+
async function requestEmbedding(text3, options, outer) {
|
|
2587
2920
|
const { baseUrl = BASE_URL, model } = options;
|
|
2588
2921
|
const timeoutMs = options.timeoutMs ?? DEFAULT_EMBEDDING_REQUEST_TIMEOUT_MS;
|
|
2589
2922
|
const headers = {
|
|
@@ -2595,7 +2928,7 @@ async function requestEmbedding(text2, options, outer) {
|
|
|
2595
2928
|
(signal) => postApiV1Embeddings({
|
|
2596
2929
|
baseUrl,
|
|
2597
2930
|
body: {
|
|
2598
|
-
input: options.maskInput ? options.maskInput(
|
|
2931
|
+
input: options.maskInput ? options.maskInput(text3) : text3,
|
|
2599
2932
|
model: model ?? DEFAULT_API_EMBEDDING_MODEL
|
|
2600
2933
|
},
|
|
2601
2934
|
headers,
|
|
@@ -2834,9 +3167,9 @@ function extractUserMessages(messages, maxCount, documentPattern) {
|
|
|
2834
3167
|
}
|
|
2835
3168
|
return result.reverse();
|
|
2836
3169
|
}
|
|
2837
|
-
function truncateAtWordBoundary(
|
|
2838
|
-
if (
|
|
2839
|
-
const truncated =
|
|
3170
|
+
function truncateAtWordBoundary(text3, maxLength) {
|
|
3171
|
+
if (text3.length <= maxLength) return text3;
|
|
3172
|
+
const truncated = text3.slice(0, maxLength);
|
|
2840
3173
|
const lastSpace = truncated.lastIndexOf(" ");
|
|
2841
3174
|
return lastSpace > maxLength * 0.5 ? truncated.slice(0, lastSpace) : truncated;
|
|
2842
3175
|
}
|
|
@@ -2961,12 +3294,12 @@ function buildSystemPrompt(ctx, options) {
|
|
|
2961
3294
|
let prompt = "";
|
|
2962
3295
|
const activeSections = [];
|
|
2963
3296
|
for (const section of sections) {
|
|
2964
|
-
const
|
|
2965
|
-
if (
|
|
2966
|
-
if (maxLength > 0 && prompt.length +
|
|
3297
|
+
const text3 = section.render(ctx);
|
|
3298
|
+
if (text3 === null) continue;
|
|
3299
|
+
if (maxLength > 0 && prompt.length + text3.length > maxLength) {
|
|
2967
3300
|
continue;
|
|
2968
3301
|
}
|
|
2969
|
-
prompt +=
|
|
3302
|
+
prompt += text3;
|
|
2970
3303
|
activeSections.push(section.key);
|
|
2971
3304
|
}
|
|
2972
3305
|
return { prompt, activeSections };
|
|
@@ -3173,15 +3506,15 @@ var PiiRedactor = class {
|
|
|
3173
3506
|
* (sequential, per-pattern) and the async {@link collectRegexSpans} (span)
|
|
3174
3507
|
* paths so the detection rules live in exactly one place.
|
|
3175
3508
|
*/
|
|
3176
|
-
findMatches(
|
|
3509
|
+
findMatches(text3, pattern) {
|
|
3177
3510
|
const regex = new RegExp(pattern.regex.source, pattern.regex.flags);
|
|
3178
3511
|
const found = [];
|
|
3179
3512
|
let m;
|
|
3180
|
-
while ((m = regex.exec(
|
|
3513
|
+
while ((m = regex.exec(text3)) !== null) {
|
|
3181
3514
|
const value = m[0];
|
|
3182
3515
|
if (pattern.validate && !pattern.validate(value)) continue;
|
|
3183
3516
|
if (pattern.context) {
|
|
3184
|
-
const before =
|
|
3517
|
+
const before = text3.slice(Math.max(0, m.index - 40), m.index);
|
|
3185
3518
|
if (!pattern.context.test(before)) continue;
|
|
3186
3519
|
}
|
|
3187
3520
|
found.push({ match: value, index: m.index });
|
|
@@ -3197,9 +3530,9 @@ var PiiRedactor = class {
|
|
|
3197
3530
|
* earlier patterns, so an earlier (more specific) pattern claims a region
|
|
3198
3531
|
* before a later one can — placeholders are inert and match no pattern.
|
|
3199
3532
|
*/
|
|
3200
|
-
scan(
|
|
3533
|
+
scan(text3, makeReplacement) {
|
|
3201
3534
|
const matches = [];
|
|
3202
|
-
let redacted =
|
|
3535
|
+
let redacted = text3;
|
|
3203
3536
|
for (const pattern of this.patterns) {
|
|
3204
3537
|
const found = this.findMatches(redacted, pattern);
|
|
3205
3538
|
if (found.length === 0) continue;
|
|
@@ -3220,8 +3553,8 @@ var PiiRedactor = class {
|
|
|
3220
3553
|
* Scan and redact PII from a single text string, using numbered, reversible
|
|
3221
3554
|
* placeholders ([EMAIL_1], …) tracked on this instance.
|
|
3222
3555
|
*/
|
|
3223
|
-
redactText(
|
|
3224
|
-
return this.scan(
|
|
3556
|
+
redactText(text3) {
|
|
3557
|
+
return this.scan(text3, (category, value) => this.getPlaceholder(category, value));
|
|
3225
3558
|
}
|
|
3226
3559
|
/**
|
|
3227
3560
|
* Mask PII with unnumbered, NON-reversible tokens ([EMAIL], [SSN], …) without
|
|
@@ -3230,15 +3563,15 @@ var PiiRedactor = class {
|
|
|
3230
3563
|
* embeddings endpoint while staying deterministic, so the same content always
|
|
3231
3564
|
* masks identically and search stays consistent across conversations.
|
|
3232
3565
|
*/
|
|
3233
|
-
maskText(
|
|
3234
|
-
return this.scan(
|
|
3566
|
+
maskText(text3) {
|
|
3567
|
+
return this.scan(text3, (category) => `[${category}]`).text;
|
|
3235
3568
|
}
|
|
3236
3569
|
/** All regex matches as spans on the ORIGINAL text (priority 0). Uses the
|
|
3237
3570
|
* same {@link findMatches} guards as the sync scan. */
|
|
3238
|
-
collectRegexSpans(
|
|
3571
|
+
collectRegexSpans(text3) {
|
|
3239
3572
|
const spans = [];
|
|
3240
3573
|
this.patterns.forEach((pattern, patternIndex) => {
|
|
3241
|
-
for (const { match, index } of this.findMatches(
|
|
3574
|
+
for (const { match, index } of this.findMatches(text3, pattern)) {
|
|
3242
3575
|
spans.push({
|
|
3243
3576
|
start: index,
|
|
3244
3577
|
end: index + match.length,
|
|
@@ -3257,16 +3590,16 @@ var PiiRedactor = class {
|
|
|
3257
3590
|
* of "Stripe") from producing a corrupting partial placeholder like
|
|
3258
3591
|
* `[ORG_1]e`.
|
|
3259
3592
|
*/
|
|
3260
|
-
normalizeNerSpans(
|
|
3593
|
+
normalizeNerSpans(text3, raw) {
|
|
3261
3594
|
if (!Array.isArray(raw)) return [];
|
|
3262
3595
|
const out = [];
|
|
3263
3596
|
raw.forEach((s, i) => {
|
|
3264
3597
|
if (!Number.isFinite(s.start) || !Number.isFinite(s.end)) return;
|
|
3265
|
-
let start = Math.max(0, Math.min(Math.trunc(s.start),
|
|
3266
|
-
let end = Math.max(0, Math.min(Math.trunc(s.end),
|
|
3598
|
+
let start = Math.max(0, Math.min(Math.trunc(s.start), text3.length));
|
|
3599
|
+
let end = Math.max(0, Math.min(Math.trunc(s.end), text3.length));
|
|
3267
3600
|
if (end <= start) return;
|
|
3268
|
-
while (start > 0 && isWordChar(
|
|
3269
|
-
while (end <
|
|
3601
|
+
while (start > 0 && isWordChar(text3[start - 1])) start--;
|
|
3602
|
+
while (end < text3.length && isWordChar(text3[end])) end++;
|
|
3270
3603
|
const category = String(s.category ?? "").trim();
|
|
3271
3604
|
if (!category) return;
|
|
3272
3605
|
out.push({ start, end, category, priority: 1, order: i });
|
|
@@ -3275,10 +3608,10 @@ var PiiRedactor = class {
|
|
|
3275
3608
|
}
|
|
3276
3609
|
/** Run the configured detector defensively: failures degrade to regex-only
|
|
3277
3610
|
* (fail-open on detection, not on protection) with a one-time warning. */
|
|
3278
|
-
async detectNerSpans(
|
|
3611
|
+
async detectNerSpans(text3) {
|
|
3279
3612
|
if (!this.nerDetector) return [];
|
|
3280
3613
|
try {
|
|
3281
|
-
return this.normalizeNerSpans(
|
|
3614
|
+
return this.normalizeNerSpans(text3, await this.nerDetector.detect(text3));
|
|
3282
3615
|
} catch (err) {
|
|
3283
3616
|
this.warnNerFailure(err);
|
|
3284
3617
|
return [];
|
|
@@ -3311,11 +3644,11 @@ var PiiRedactor = class {
|
|
|
3311
3644
|
* in pattern then document order, matching the sync scan's numbering). The
|
|
3312
3645
|
* string is then EMITTED in document order.
|
|
3313
3646
|
*/
|
|
3314
|
-
rebuildSpans(
|
|
3647
|
+
rebuildSpans(text3, spans, makeReplacement) {
|
|
3315
3648
|
const placeholderByStart = /* @__PURE__ */ new Map();
|
|
3316
3649
|
const matches = [];
|
|
3317
3650
|
for (const s of spans) {
|
|
3318
|
-
const value =
|
|
3651
|
+
const value = text3.slice(s.start, s.end);
|
|
3319
3652
|
const placeholder = makeReplacement(s.category, value);
|
|
3320
3653
|
placeholderByStart.set(s.start, placeholder);
|
|
3321
3654
|
matches.push({ category: s.category, original: value, placeholder });
|
|
@@ -3324,16 +3657,16 @@ var PiiRedactor = class {
|
|
|
3324
3657
|
let result = "";
|
|
3325
3658
|
let cursor = 0;
|
|
3326
3659
|
for (const s of emitOrder) {
|
|
3327
|
-
result +=
|
|
3660
|
+
result += text3.slice(cursor, s.start) + (placeholderByStart.get(s.start) ?? "");
|
|
3328
3661
|
cursor = s.end;
|
|
3329
3662
|
}
|
|
3330
|
-
result +=
|
|
3663
|
+
result += text3.slice(cursor);
|
|
3331
3664
|
return { text: result, matches };
|
|
3332
3665
|
}
|
|
3333
3666
|
/** Merge regex + NER spans for `text` into a resolved, non-overlapping set. */
|
|
3334
|
-
async detectAllSpans(
|
|
3335
|
-
const nerPromise = this.detectNerSpans(
|
|
3336
|
-
const regexSpans = this.collectRegexSpans(
|
|
3667
|
+
async detectAllSpans(text3) {
|
|
3668
|
+
const nerPromise = this.detectNerSpans(text3);
|
|
3669
|
+
const regexSpans = this.collectRegexSpans(text3);
|
|
3337
3670
|
const ner = await nerPromise;
|
|
3338
3671
|
return this.resolveOverlaps([...regexSpans, ...ner]);
|
|
3339
3672
|
}
|
|
@@ -3342,11 +3675,11 @@ var PiiRedactor = class {
|
|
|
3342
3675
|
* unstructured PII (the configured {@link NerDetector}) with numbered,
|
|
3343
3676
|
* reversible placeholders. With no detector, identical to `redactText`.
|
|
3344
3677
|
*/
|
|
3345
|
-
async redactTextAsync(
|
|
3346
|
-
if (!this.nerDetector) return this.redactText(
|
|
3347
|
-
const spans = await this.detectAllSpans(
|
|
3678
|
+
async redactTextAsync(text3) {
|
|
3679
|
+
if (!this.nerDetector) return this.redactText(text3);
|
|
3680
|
+
const spans = await this.detectAllSpans(text3);
|
|
3348
3681
|
return this.rebuildSpans(
|
|
3349
|
-
|
|
3682
|
+
text3,
|
|
3350
3683
|
spans,
|
|
3351
3684
|
(category, value) => this.getPlaceholder(category, value)
|
|
3352
3685
|
);
|
|
@@ -3355,10 +3688,10 @@ var PiiRedactor = class {
|
|
|
3355
3688
|
* Async counterpart to {@link maskText}: stateless `[CATEGORY]` masking over
|
|
3356
3689
|
* regex + NER matches. With no detector, identical to `maskText`.
|
|
3357
3690
|
*/
|
|
3358
|
-
async maskTextAsync(
|
|
3359
|
-
if (!this.nerDetector) return this.maskText(
|
|
3360
|
-
const spans = await this.detectAllSpans(
|
|
3361
|
-
return this.rebuildSpans(
|
|
3691
|
+
async maskTextAsync(text3) {
|
|
3692
|
+
if (!this.nerDetector) return this.maskText(text3);
|
|
3693
|
+
const spans = await this.detectAllSpans(text3);
|
|
3694
|
+
return this.rebuildSpans(text3, spans, (category) => `[${category}]`).text;
|
|
3362
3695
|
}
|
|
3363
3696
|
/**
|
|
3364
3697
|
* Async counterpart to {@link redactMessages}. Text parts are redacted
|
|
@@ -3444,9 +3777,9 @@ var PiiRedactor = class {
|
|
|
3444
3777
|
* {@link restoreForStorage} instead, which tolerates the mangled echoes the
|
|
3445
3778
|
* extraction models produce.
|
|
3446
3779
|
*/
|
|
3447
|
-
deAnonymize(
|
|
3448
|
-
if (!
|
|
3449
|
-
let restored =
|
|
3780
|
+
deAnonymize(text3) {
|
|
3781
|
+
if (!text3.includes("[")) return text3;
|
|
3782
|
+
let restored = text3;
|
|
3450
3783
|
for (const [placeholder, original] of this.placeholderToValue) {
|
|
3451
3784
|
if (restored.includes(placeholder)) {
|
|
3452
3785
|
restored = restored.split(placeholder).join(original);
|
|
@@ -3471,15 +3804,15 @@ var PiiRedactor = class {
|
|
|
3471
3804
|
* - `unresolved` is derived from the lookup miss during this pass, not from a
|
|
3472
3805
|
* regex re-scan of the output — so it can't trip over a restored value.
|
|
3473
3806
|
*/
|
|
3474
|
-
restoreForStorage(
|
|
3807
|
+
restoreForStorage(text3) {
|
|
3475
3808
|
if (this.storagePattern === void 0) this.storagePattern = this.buildStoragePattern();
|
|
3476
3809
|
if (!this.storagePattern || this.placeholderToValue.size === 0) {
|
|
3477
|
-
return { text:
|
|
3810
|
+
return { text: text3, unresolved: false };
|
|
3478
3811
|
}
|
|
3479
3812
|
this.storageLookup ?? (this.storageLookup = this.buildStorageLookup());
|
|
3480
3813
|
const { exact, ci, ciCollisions } = this.storageLookup;
|
|
3481
3814
|
let unresolved = false;
|
|
3482
|
-
const restored =
|
|
3815
|
+
const restored = text3.replace(
|
|
3483
3816
|
this.storagePattern,
|
|
3484
3817
|
(match, bracketed, bare) => {
|
|
3485
3818
|
const body = bracketed ?? bare ?? "";
|
|
@@ -3975,7 +4308,7 @@ function findMatchingTools(promptEmbeddings, tools, options) {
|
|
|
3975
4308
|
if (!tools || tools.length === 0) {
|
|
3976
4309
|
return [];
|
|
3977
4310
|
}
|
|
3978
|
-
const
|
|
4311
|
+
const embeddings2 = Array.isArray(promptEmbeddings[0]) ? promptEmbeddings : [promptEmbeddings];
|
|
3979
4312
|
const results = [];
|
|
3980
4313
|
for (const tool of tools) {
|
|
3981
4314
|
if (!tool.embedding || tool.embedding.length === 0) {
|
|
@@ -3983,7 +4316,7 @@ function findMatchingTools(promptEmbeddings, tools, options) {
|
|
|
3983
4316
|
}
|
|
3984
4317
|
try {
|
|
3985
4318
|
let maxSimilarity = -Infinity;
|
|
3986
|
-
for (const embedding of
|
|
4319
|
+
for (const embedding of embeddings2) {
|
|
3987
4320
|
const similarity = cosineSimilarity(embedding, tool.embedding);
|
|
3988
4321
|
if (similarity > maxSimilarity) {
|
|
3989
4322
|
maxSimilarity = similarity;
|
|
@@ -4014,12 +4347,12 @@ function scoreTools(promptEmbeddings, tools) {
|
|
|
4014
4347
|
const scores = /* @__PURE__ */ new Map();
|
|
4015
4348
|
if (!promptEmbeddings || promptEmbeddings.length === 0) return scores;
|
|
4016
4349
|
if (!tools || tools.length === 0) return scores;
|
|
4017
|
-
const
|
|
4350
|
+
const embeddings2 = Array.isArray(promptEmbeddings[0]) ? promptEmbeddings : [promptEmbeddings];
|
|
4018
4351
|
for (const tool of tools) {
|
|
4019
4352
|
if (!tool.embedding || tool.embedding.length === 0) continue;
|
|
4020
4353
|
let maxSimilarity = -Infinity;
|
|
4021
4354
|
try {
|
|
4022
|
-
for (const embedding of
|
|
4355
|
+
for (const embedding of embeddings2) {
|
|
4023
4356
|
const similarity = cosineSimilarity(embedding, tool.embedding);
|
|
4024
4357
|
if (similarity > maxSimilarity) maxSimilarity = similarity;
|
|
4025
4358
|
}
|
|
@@ -4241,13 +4574,13 @@ function createServerToolsFilter(options = {}) {
|
|
|
4241
4574
|
const exclude = new Set(options.excludeTools ?? []);
|
|
4242
4575
|
const sets = options.toolSets ?? [];
|
|
4243
4576
|
const matchOpts = options.matchOptions;
|
|
4244
|
-
const filter = (
|
|
4245
|
-
const matches = findMatchingTools(
|
|
4577
|
+
const filter = (embeddings2, tools) => {
|
|
4578
|
+
const matches = findMatchingTools(embeddings2, tools, matchOpts);
|
|
4246
4579
|
if (matches.length === 0) return [];
|
|
4247
4580
|
const matchedNames = new Set(matches.map((m) => m.tool.name));
|
|
4248
4581
|
let finalNames;
|
|
4249
4582
|
if (sets.length > 0) {
|
|
4250
|
-
const scores = scoreTools(
|
|
4583
|
+
const scores = scoreTools(embeddings2, tools);
|
|
4251
4584
|
const selectedScores = new Map(
|
|
4252
4585
|
[...scores].filter(([name]) => matchedNames.has(name) && !exclude.has(name))
|
|
4253
4586
|
);
|