@anuma/sdk 9.1.11 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/index.cjs +573 -239
- package/dist/expo/index.d.mts +86 -1
- package/dist/expo/index.d.ts +86 -1
- package/dist/expo/index.mjs +599 -265
- package/dist/index.cjs +444 -111
- package/dist/index.d.mts +86 -1
- package/dist/index.d.ts +86 -1
- package/dist/index.mjs +444 -111
- package/dist/react/index.cjs +620 -286
- package/dist/react/index.d.mts +86 -1
- package/dist/react/index.d.ts +86 -1
- package/dist/react/index.mjs +646 -312
- package/dist/server/index.cjs +1 -0
- package/dist/server/index.mjs +1 -0
- package/dist/tools/index.cjs +19 -13
- package/dist/tools/index.d.mts +8 -12
- package/dist/tools/index.d.ts +8 -12
- package/dist/tools/index.mjs +19 -13
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -684,8 +684,8 @@ var createClient = (config = {}) => {
|
|
|
684
684
|
data = await response[parseAs]();
|
|
685
685
|
break;
|
|
686
686
|
case "json": {
|
|
687
|
-
const
|
|
688
|
-
data =
|
|
687
|
+
const text3 = await response.text();
|
|
688
|
+
data = text3 ? JSON.parse(text3) : {};
|
|
689
689
|
break;
|
|
690
690
|
}
|
|
691
691
|
case "stream":
|
|
@@ -1125,8 +1125,8 @@ var ATTACHED_FILES_OPEN_TAG = "<attached_files>";
|
|
|
1125
1125
|
var ATTACHED_FILES_CLOSE_TAG = "</attached_files>";
|
|
1126
1126
|
var ATTACHED_FILES_PREFIX = `${ATTACHED_FILES_OPEN_TAG}
|
|
1127
1127
|
The user attached the following file(s) to this message. Their extracted contents are below; when the user refers to "the attached file", "this file" or "the document", they mean these.`;
|
|
1128
|
-
function isAttachedFilesText(
|
|
1129
|
-
return typeof
|
|
1128
|
+
function isAttachedFilesText(text3) {
|
|
1129
|
+
return typeof text3 === "string" && text3.startsWith(ATTACHED_FILES_PREFIX) && text3.endsWith(ATTACHED_FILES_CLOSE_TAG);
|
|
1130
1130
|
}
|
|
1131
1131
|
function buildAttachedFilesText(fileContext) {
|
|
1132
1132
|
return `${ATTACHED_FILES_PREFIX}
|
|
@@ -1739,6 +1739,7 @@ function createNearbyRequest(options, apiPath = "/api/private-memories") {
|
|
|
1739
1739
|
}
|
|
1740
1740
|
throw error;
|
|
1741
1741
|
}
|
|
1742
|
+
if (response.status === 204) return null;
|
|
1742
1743
|
return await response.json();
|
|
1743
1744
|
} catch (error) {
|
|
1744
1745
|
if (timedOut) throw new RemoteMemoryError("Nearby request timed out", 408, "timeout");
|
|
@@ -1766,6 +1767,16 @@ var ChatStoreError = class extends Error {
|
|
|
1766
1767
|
// src/lib/chatStore/remote.ts
|
|
1767
1768
|
var kinds = /* @__PURE__ */ new Set(["conversation", "message", "summary", "project", "memory_links"]);
|
|
1768
1769
|
var cipherPattern = /^enc:v\d+:(?:[0-9a-f]{2}){28,}$/i;
|
|
1770
|
+
var maxPayload = 2 * 1024 * 1024;
|
|
1771
|
+
var storedDims = 1536;
|
|
1772
|
+
var maxRequestCiphertext = 8 * 1024 * 1024;
|
|
1773
|
+
var maxGetRefs = 50;
|
|
1774
|
+
var maxImportRecords = 100;
|
|
1775
|
+
var maxVerifyRevisions = 1e3;
|
|
1776
|
+
var maxChunksPerRecord = 16;
|
|
1777
|
+
var maxChunksPerRequest = 200;
|
|
1778
|
+
var maxAttachmentBytes = 50 * 1024 * 1024;
|
|
1779
|
+
var attachmentIdPattern = /^[0-9a-f]{32}$/;
|
|
1769
1780
|
var metadata = [
|
|
1770
1781
|
"kind",
|
|
1771
1782
|
"id",
|
|
@@ -1808,7 +1819,62 @@ function validRecord(value) {
|
|
|
1808
1819
|
))
|
|
1809
1820
|
return false;
|
|
1810
1821
|
const root = value.kind === "conversation" || value.kind === "project";
|
|
1811
|
-
return root ? value.conversationId === ""
|
|
1822
|
+
return root ? value.conversationId === "" : id(value.conversationId);
|
|
1823
|
+
}
|
|
1824
|
+
function embeddings(write) {
|
|
1825
|
+
if (write.embeddings === void 0) return true;
|
|
1826
|
+
return Array.isArray(write.embeddings) && !write.record.isDeleted && (write.record.kind === "message" || write.record.kind === "summary") && write.embeddings.length <= maxChunksPerRecord && (write.embeddingModel === void 0 || typeof write.embeddingModel === "string" && write.embeddingModel.length <= 256) && Array.from({ length: write.embeddings.length }, (_, i) => i).every(
|
|
1827
|
+
(i) => vector(write.embeddings?.[i])
|
|
1828
|
+
);
|
|
1829
|
+
}
|
|
1830
|
+
function vector(v) {
|
|
1831
|
+
if (!Array.isArray(v) || v.length !== 1536 && v.length !== 4096) return false;
|
|
1832
|
+
let nonZero = false;
|
|
1833
|
+
for (let i = 0; i < v.length; i++) {
|
|
1834
|
+
const x = v[i];
|
|
1835
|
+
if (typeof x !== "number" || !Number.isFinite(x)) return false;
|
|
1836
|
+
if (i < storedDims && x !== 0) nonZero = true;
|
|
1837
|
+
}
|
|
1838
|
+
return nonZero;
|
|
1839
|
+
}
|
|
1840
|
+
function vectorFields(write) {
|
|
1841
|
+
return {
|
|
1842
|
+
...write.embeddings !== void 0 ? { embeddings: write.embeddings.map((v) => v.slice(0, storedDims)) } : {},
|
|
1843
|
+
...write.embeddingModel !== void 0 ? { embedding_model: write.embeddingModel } : {}
|
|
1844
|
+
};
|
|
1845
|
+
}
|
|
1846
|
+
function text2(value) {
|
|
1847
|
+
return typeof value === "string";
|
|
1848
|
+
}
|
|
1849
|
+
function timestamp(value) {
|
|
1850
|
+
return typeof value === "string" && Number.isFinite(Date.parse(value));
|
|
1851
|
+
}
|
|
1852
|
+
function presigned(value) {
|
|
1853
|
+
if (!object(value) || !text2(value.url) || !/^https:\/\//.test(value.url) || !text2(value.method) || !object(value.headers) || !Object.values(value.headers).every(text2) || !timestamp(value.expires_at))
|
|
1854
|
+
invalid();
|
|
1855
|
+
return {
|
|
1856
|
+
url: value.url,
|
|
1857
|
+
method: value.method,
|
|
1858
|
+
headers: value.headers,
|
|
1859
|
+
expiresAt: value.expires_at
|
|
1860
|
+
};
|
|
1861
|
+
}
|
|
1862
|
+
function attachment(value) {
|
|
1863
|
+
if (!object(value) || !text2(value.attachment_id) || !attachmentIdPattern.test(value.attachment_id) || !integer(value.size_bytes, 1) || value.state !== "pending" && value.state !== "committed" || value.owner_kind !== void 0 && value.owner_kind !== "message" && value.owner_kind !== "summary" || value.owner_id !== void 0 && !id(value.owner_id) || !timestamp(value.created_at) || value.committed_at !== void 0 && !timestamp(value.committed_at))
|
|
1864
|
+
invalid();
|
|
1865
|
+
return {
|
|
1866
|
+
id: value.attachment_id,
|
|
1867
|
+
sizeBytes: value.size_bytes,
|
|
1868
|
+
state: value.state,
|
|
1869
|
+
...value.owner_kind !== void 0 ? { ownerKind: value.owner_kind } : {},
|
|
1870
|
+
...value.owner_id !== void 0 ? { ownerId: value.owner_id } : {},
|
|
1871
|
+
createdAt: value.created_at,
|
|
1872
|
+
...value.committed_at !== void 0 ? { committedAt: value.committed_at } : {}
|
|
1873
|
+
};
|
|
1874
|
+
}
|
|
1875
|
+
function ref(value) {
|
|
1876
|
+
if (!object(value) || !text2(value.kind) || !kinds.has(value.kind) || !id(value.id)) invalid();
|
|
1877
|
+
return { kind: value.kind, id: value.id };
|
|
1812
1878
|
}
|
|
1813
1879
|
function sameMetadata(a, b) {
|
|
1814
1880
|
return metadata.every((key) => a[key] === b[key]);
|
|
@@ -1864,8 +1930,11 @@ async function createRemoteChatStore(options) {
|
|
|
1864
1930
|
isDeleted: r.is_deleted,
|
|
1865
1931
|
data: {}
|
|
1866
1932
|
};
|
|
1867
|
-
if (!validRecord(plain) || typeof r.payload !== "string" || r.payload.length >
|
|
1933
|
+
if (!validRecord(plain) || typeof r.payload !== "string" || r.payload.length > maxPayload)
|
|
1868
1934
|
invalid();
|
|
1935
|
+
if (plain.isDeleted && r.payload === "")
|
|
1936
|
+
return { record: plain, version: value.version, serverUpdatedAt: value.server_updated_at };
|
|
1937
|
+
if (!cipherPattern.test(r.payload)) invalid();
|
|
1869
1938
|
const envelope = JSON.parse(await decrypt(r.payload));
|
|
1870
1939
|
if (!object(envelope) || envelope.schema !== 1 || envelope.account_id !== accountId || envelope.key_id !== keyId || !validRecord(envelope.record) || !sameMetadata(plain, envelope.record))
|
|
1871
1940
|
invalid();
|
|
@@ -1875,30 +1944,46 @@ async function createRemoteChatStore(options) {
|
|
|
1875
1944
|
serverUpdatedAt: value.server_updated_at
|
|
1876
1945
|
};
|
|
1877
1946
|
};
|
|
1878
|
-
const
|
|
1879
|
-
if (
|
|
1947
|
+
const seal = async (record2, signal) => {
|
|
1948
|
+
if (record2.isDeleted) return "";
|
|
1949
|
+
const payload = await encrypt(
|
|
1950
|
+
JSON.stringify({ schema: 1, account_id: accountId, key_id: keyId, record: record2 })
|
|
1951
|
+
);
|
|
1952
|
+
signal?.throwIfAborted();
|
|
1953
|
+
if (!cipherPattern.test(payload) || payload.length > maxPayload) invalid();
|
|
1954
|
+
return payload;
|
|
1955
|
+
};
|
|
1956
|
+
const capture = (items, max, signal) => {
|
|
1957
|
+
if (!Array.isArray(items) || items.length < 1 || items.length > max) invalid();
|
|
1880
1958
|
const seen = /* @__PURE__ */ new Set();
|
|
1881
|
-
|
|
1959
|
+
let chunks = 0;
|
|
1960
|
+
return items.map((item) => {
|
|
1882
1961
|
signal?.throwIfAborted();
|
|
1883
|
-
if (!validRecord(
|
|
1884
|
-
|
|
1885
|
-
|
|
1962
|
+
if (!validRecord(item.record) || !embeddings(item)) invalid();
|
|
1963
|
+
chunks += item.embeddings?.length ?? 0;
|
|
1964
|
+
if (chunks > maxChunksPerRequest) invalid();
|
|
1965
|
+
const copy = JSON.parse(JSON.stringify(item));
|
|
1966
|
+
const recordKey = `${copy.record.kind}/${copy.record.id}`;
|
|
1886
1967
|
if (seen.has(recordKey)) invalid();
|
|
1887
1968
|
seen.add(recordKey);
|
|
1888
|
-
return
|
|
1969
|
+
return copy;
|
|
1889
1970
|
});
|
|
1971
|
+
};
|
|
1972
|
+
const putMany = async (writes, signal) => {
|
|
1973
|
+
if (Array.isArray(writes) && writes.some((w) => !integer(w.expectedVersion)))
|
|
1974
|
+
invalid();
|
|
1975
|
+
const captured = capture(writes, 25, signal);
|
|
1890
1976
|
let bytes = 0;
|
|
1891
1977
|
const prepared = [];
|
|
1892
1978
|
for (const write of captured) {
|
|
1893
|
-
const
|
|
1894
|
-
const payload = await encrypt(
|
|
1895
|
-
JSON.stringify({ schema: 1, account_id: accountId, key_id: keyId, record: record2 })
|
|
1896
|
-
);
|
|
1897
|
-
signal?.throwIfAborted();
|
|
1898
|
-
if (!cipherPattern.test(payload) || payload.length > 2 * 1024 * 1024) invalid();
|
|
1979
|
+
const payload = await seal(write.record, signal);
|
|
1899
1980
|
bytes += payload.length;
|
|
1900
|
-
if (bytes >
|
|
1901
|
-
prepared.push({
|
|
1981
|
+
if (bytes > maxRequestCiphertext) invalid();
|
|
1982
|
+
prepared.push({
|
|
1983
|
+
record: wire(write.record, payload),
|
|
1984
|
+
expected_version: write.expectedVersion,
|
|
1985
|
+
...vectorFields(write)
|
|
1986
|
+
});
|
|
1902
1987
|
}
|
|
1903
1988
|
const result = await request(
|
|
1904
1989
|
"/records/batch",
|
|
@@ -1913,7 +1998,8 @@ async function createRemoteChatStore(options) {
|
|
|
1913
1998
|
const out = [];
|
|
1914
1999
|
for (let i = 0; i < result.items.length; i++) {
|
|
1915
2000
|
const value = result.items[i];
|
|
1916
|
-
|
|
2001
|
+
const deleted = prepared[i].record.is_deleted;
|
|
2002
|
+
if (!object(value) || !object(value.record) || value.record.kind !== prepared[i].record.kind || value.record.id !== prepared[i].record.id || value.record.payload !== prepared[i].record.payload || (deleted ? value.record.is_deleted !== true || !integer(value.version, 1) : value.version !== prepared[i].expected_version + 1))
|
|
1917
2003
|
invalid();
|
|
1918
2004
|
out.push(await decode(value));
|
|
1919
2005
|
}
|
|
@@ -1942,7 +2028,7 @@ async function createRemoteChatStore(options) {
|
|
|
1942
2028
|
async list(query) {
|
|
1943
2029
|
const limit = query.limit ?? 20;
|
|
1944
2030
|
const parent = query.conversationId ?? "";
|
|
1945
|
-
if (!kinds.has(query.kind) || !integer(limit, 1) || query.includeDeleted !== void 0 && typeof query.includeDeleted !== "boolean" || limit > 50 || (query.cursor?.length ?? 0) > 2048 || (query.kind === "conversation" || query.kind === "project" ? parent !== "" : !id(parent)))
|
|
2031
|
+
if (!kinds.has(query.kind) || !integer(limit, 1) || query.includeDeleted !== void 0 && typeof query.includeDeleted !== "boolean" || limit > (query.kind === "message" ? 200 : 50) || (query.cursor?.length ?? 0) > 2048 || (query.kind === "conversation" || query.kind === "project" ? parent !== "" : !id(parent)))
|
|
1946
2032
|
invalid();
|
|
1947
2033
|
const params = new URLSearchParams({
|
|
1948
2034
|
account_id: String(accountId),
|
|
@@ -1953,7 +2039,7 @@ async function createRemoteChatStore(options) {
|
|
|
1953
2039
|
});
|
|
1954
2040
|
if (query.cursor) params.set("cursor", query.cursor);
|
|
1955
2041
|
const value = await request(`/records?${params}`, {}, query.signal);
|
|
1956
|
-
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit || value.next_cursor !== void 0 && (typeof value.next_cursor !== "string" || value.next_cursor.length > 2048))
|
|
2042
|
+
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit || value.next_cursor !== void 0 && (typeof value.next_cursor !== "string" || value.next_cursor.length > 2048) || value.total !== void 0 && !integer(value.total))
|
|
1957
2043
|
invalid();
|
|
1958
2044
|
const items = [];
|
|
1959
2045
|
for (const item of value.items) {
|
|
@@ -1963,11 +2049,258 @@ async function createRemoteChatStore(options) {
|
|
|
1963
2049
|
items.push(result);
|
|
1964
2050
|
}
|
|
1965
2051
|
query.signal?.throwIfAborted();
|
|
1966
|
-
return {
|
|
2052
|
+
return {
|
|
2053
|
+
items,
|
|
2054
|
+
...value.next_cursor ? { nextCursor: value.next_cursor } : {},
|
|
2055
|
+
...value.total !== void 0 ? { total: value.total } : {}
|
|
2056
|
+
};
|
|
2057
|
+
},
|
|
2058
|
+
putMany,
|
|
2059
|
+
async getMany(refs, signal) {
|
|
2060
|
+
if (!Array.isArray(refs)) invalid();
|
|
2061
|
+
const wanted = /* @__PURE__ */ new Map();
|
|
2062
|
+
for (const r of refs) {
|
|
2063
|
+
const checked = ref(r);
|
|
2064
|
+
wanted.set(`${checked.kind}/${checked.id}`, checked);
|
|
2065
|
+
}
|
|
2066
|
+
const found = /* @__PURE__ */ new Map();
|
|
2067
|
+
let pending = [...wanted.values()];
|
|
2068
|
+
while (pending.length > 0) {
|
|
2069
|
+
const batch = pending.slice(0, maxGetRefs);
|
|
2070
|
+
pending = pending.slice(maxGetRefs);
|
|
2071
|
+
const value = await request(
|
|
2072
|
+
"/records/get",
|
|
2073
|
+
{ method: "POST", body: JSON.stringify({ account_id: accountId, refs: batch }) },
|
|
2074
|
+
signal
|
|
2075
|
+
);
|
|
2076
|
+
if (!object(value) || !Array.isArray(value.items) || value.remaining !== void 0 && !Array.isArray(value.remaining))
|
|
2077
|
+
invalid();
|
|
2078
|
+
const asked = new Set(batch.map((r) => `${r.kind}/${r.id}`));
|
|
2079
|
+
for (const item of value.items) {
|
|
2080
|
+
const snapshot = await decode(item);
|
|
2081
|
+
const key = `${snapshot.record.kind}/${snapshot.record.id}`;
|
|
2082
|
+
if (!asked.has(key) || found.has(key)) invalid();
|
|
2083
|
+
found.set(key, snapshot);
|
|
2084
|
+
}
|
|
2085
|
+
const remaining = (value.remaining ?? []).map(ref);
|
|
2086
|
+
if (remaining.some((r) => !asked.has(`${r.kind}/${r.id}`) || found.has(`${r.kind}/${r.id}`)))
|
|
2087
|
+
invalid();
|
|
2088
|
+
if (remaining.length > 0 && value.items.length === 0) invalid();
|
|
2089
|
+
pending = [...remaining, ...pending];
|
|
2090
|
+
}
|
|
2091
|
+
signal?.throwIfAborted();
|
|
2092
|
+
return [...wanted.keys()].flatMap((key) => {
|
|
2093
|
+
const snapshot = found.get(key);
|
|
2094
|
+
return snapshot ? [snapshot] : [];
|
|
2095
|
+
});
|
|
2096
|
+
},
|
|
2097
|
+
async account(signal) {
|
|
2098
|
+
const value = await request("/account", {}, signal);
|
|
2099
|
+
if (!object(value) || value.account_id !== accountId || !text2(value.key_id) || value.state !== "none" && value.state !== "migrating" && value.state !== "active" || !integer(value.record_count) || !integer(value.storage_bytes) || !integer(value.vector_count))
|
|
2100
|
+
invalid();
|
|
2101
|
+
if (value.key_id && value.key_id !== keyId)
|
|
2102
|
+
throw new ChatStoreError("Canonical chat key does not match", 409, "key_mismatch");
|
|
2103
|
+
const out = {
|
|
2104
|
+
keyId: value.key_id,
|
|
2105
|
+
state: value.state,
|
|
2106
|
+
recordCount: value.record_count,
|
|
2107
|
+
storageBytes: value.storage_bytes,
|
|
2108
|
+
vectorCount: value.vector_count
|
|
2109
|
+
};
|
|
2110
|
+
return out;
|
|
2111
|
+
},
|
|
2112
|
+
async importRecords(records, signal) {
|
|
2113
|
+
const captured = capture(records, maxImportRecords, signal);
|
|
2114
|
+
let bytes = 0;
|
|
2115
|
+
const prepared = [];
|
|
2116
|
+
for (const item of captured) {
|
|
2117
|
+
const payload = await seal(item.record, signal);
|
|
2118
|
+
bytes += payload.length;
|
|
2119
|
+
if (bytes > maxRequestCiphertext) invalid();
|
|
2120
|
+
prepared.push({ record: wire(item.record, payload), ...vectorFields(item) });
|
|
2121
|
+
}
|
|
2122
|
+
const value = await request(
|
|
2123
|
+
"/import",
|
|
2124
|
+
{
|
|
2125
|
+
method: "POST",
|
|
2126
|
+
body: JSON.stringify({ account_id: accountId, key_id: keyId, records: prepared })
|
|
2127
|
+
},
|
|
2128
|
+
signal
|
|
2129
|
+
);
|
|
2130
|
+
if (!object(value) || !integer(value.written) || value.written > prepared.length) invalid();
|
|
2131
|
+
return value.written;
|
|
2132
|
+
},
|
|
2133
|
+
async verify(revisions, signal) {
|
|
2134
|
+
if (!Array.isArray(revisions) || revisions.length > maxVerifyRevisions) invalid();
|
|
2135
|
+
const asked = /* @__PURE__ */ new Set();
|
|
2136
|
+
const wireRevisions = revisions.map((r) => {
|
|
2137
|
+
const checked = ref(r);
|
|
2138
|
+
if (!integer(r.updatedAt, 1) || typeof r.isDeleted !== "boolean") invalid();
|
|
2139
|
+
asked.add(`${checked.kind}/${checked.id}`);
|
|
2140
|
+
return {
|
|
2141
|
+
kind: checked.kind,
|
|
2142
|
+
id: checked.id,
|
|
2143
|
+
updated_at: r.updatedAt,
|
|
2144
|
+
is_deleted: r.isDeleted
|
|
2145
|
+
};
|
|
2146
|
+
});
|
|
2147
|
+
if (wireRevisions.length === 0) return [];
|
|
2148
|
+
const value = await request(
|
|
2149
|
+
"/verify",
|
|
2150
|
+
{
|
|
2151
|
+
method: "POST",
|
|
2152
|
+
body: JSON.stringify({ account_id: accountId, revisions: wireRevisions })
|
|
2153
|
+
},
|
|
2154
|
+
signal
|
|
2155
|
+
);
|
|
2156
|
+
if (!object(value) || !Array.isArray(value.missing)) invalid();
|
|
2157
|
+
const missing = value.missing.map(ref);
|
|
2158
|
+
if (missing.some((r) => !asked.has(`${r.kind}/${r.id}`))) invalid();
|
|
2159
|
+
return missing;
|
|
2160
|
+
},
|
|
2161
|
+
async activate(signal) {
|
|
2162
|
+
const value = await request(
|
|
2163
|
+
"/account",
|
|
2164
|
+
{ method: "PUT", body: JSON.stringify({ account_id: accountId, state: "active" }) },
|
|
2165
|
+
signal
|
|
2166
|
+
);
|
|
2167
|
+
if (!object(value) || value.state !== "active" || typeof value.transitioned !== "boolean")
|
|
2168
|
+
invalid();
|
|
2169
|
+
return value.transitioned;
|
|
2170
|
+
},
|
|
2171
|
+
async changes(cursor, options2) {
|
|
2172
|
+
const limit = options2?.limit ?? 100;
|
|
2173
|
+
if (!integer(limit, 1) || limit > 200 || cursor !== void 0 && (!text2(cursor) || cursor.length > 2048))
|
|
2174
|
+
invalid();
|
|
2175
|
+
const params = new URLSearchParams({ account_id: String(accountId), limit: String(limit) });
|
|
2176
|
+
if (cursor) params.set("cursor", cursor);
|
|
2177
|
+
const value = await request(`/changes?${params}`, {}, options2?.signal);
|
|
2178
|
+
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit || value.next_cursor !== void 0 && !text2(value.next_cursor) || value.changes_cursor !== void 0 && !text2(value.changes_cursor))
|
|
2179
|
+
invalid();
|
|
2180
|
+
const items = [];
|
|
2181
|
+
for (const item of value.items) items.push(await decode(item));
|
|
2182
|
+
options2?.signal?.throwIfAborted();
|
|
2183
|
+
return {
|
|
2184
|
+
items,
|
|
2185
|
+
...value.next_cursor ? { nextCursor: value.next_cursor } : {},
|
|
2186
|
+
changesCursor: value.changes_cursor ?? cursor ?? ""
|
|
2187
|
+
};
|
|
2188
|
+
},
|
|
2189
|
+
async search(embedding, options2) {
|
|
2190
|
+
const limit = options2?.limit ?? 20;
|
|
2191
|
+
const kindsFilter = options2?.kinds;
|
|
2192
|
+
if (!Array.isArray(embedding) || !embeddings({
|
|
2193
|
+
record: { ...emptyMessage, isDeleted: false },
|
|
2194
|
+
embeddings: [embedding]
|
|
2195
|
+
}) || !integer(limit, 1) || limit > 50 || kindsFilter !== void 0 && (!Array.isArray(kindsFilter) || kindsFilter.length === 0 || kindsFilter.some((k) => k !== "message" && k !== "summary")) || options2?.conversationId !== void 0 && !id(options2.conversationId))
|
|
2196
|
+
invalid();
|
|
2197
|
+
const value = await request(
|
|
2198
|
+
"/search",
|
|
2199
|
+
{
|
|
2200
|
+
method: "POST",
|
|
2201
|
+
body: JSON.stringify({
|
|
2202
|
+
account_id: accountId,
|
|
2203
|
+
embedding: embedding.slice(0, storedDims),
|
|
2204
|
+
limit,
|
|
2205
|
+
...kindsFilter ? { kinds: kindsFilter } : {},
|
|
2206
|
+
...options2?.conversationId ? { conversation_id: options2.conversationId } : {}
|
|
2207
|
+
})
|
|
2208
|
+
},
|
|
2209
|
+
options2?.signal
|
|
2210
|
+
);
|
|
2211
|
+
if (!object(value) || !Array.isArray(value.items) || value.items.length > limit) invalid();
|
|
2212
|
+
const hits = [];
|
|
2213
|
+
for (const item of value.items) {
|
|
2214
|
+
if (!object(item) || !integer(item.chunk) || typeof item.score !== "number" || !Number.isFinite(item.score))
|
|
2215
|
+
invalid();
|
|
2216
|
+
const snapshot = await decode(item);
|
|
2217
|
+
if (snapshot.record.isDeleted || kindsFilter !== void 0 && !kindsFilter.includes(snapshot.record.kind) || snapshot.record.kind !== "message" && snapshot.record.kind !== "summary" || options2?.conversationId !== void 0 && snapshot.record.conversationId !== options2.conversationId)
|
|
2218
|
+
invalid();
|
|
2219
|
+
hits.push({ ...snapshot, chunk: item.chunk, score: item.score });
|
|
2220
|
+
}
|
|
2221
|
+
options2?.signal?.throwIfAborted();
|
|
2222
|
+
return hits;
|
|
2223
|
+
},
|
|
2224
|
+
async deleteAll(signal) {
|
|
2225
|
+
const value = await request(`/records?account_id=${accountId}`, { method: "DELETE" }, signal);
|
|
2226
|
+
if (!object(value) || !integer(value.deleted)) invalid();
|
|
2227
|
+
return value.deleted;
|
|
1967
2228
|
},
|
|
1968
|
-
|
|
2229
|
+
attachments: {
|
|
2230
|
+
async create(sizeBytes, signal) {
|
|
2231
|
+
if (!integer(sizeBytes, 1) || sizeBytes > maxAttachmentBytes) invalid();
|
|
2232
|
+
const value = await request(
|
|
2233
|
+
"/attachments",
|
|
2234
|
+
{
|
|
2235
|
+
method: "POST",
|
|
2236
|
+
body: JSON.stringify({ account_id: accountId, size_bytes: sizeBytes })
|
|
2237
|
+
},
|
|
2238
|
+
signal
|
|
2239
|
+
);
|
|
2240
|
+
if (!object(value)) invalid();
|
|
2241
|
+
const created = attachment(value.attachment);
|
|
2242
|
+
if (created.sizeBytes !== sizeBytes || created.state !== "pending") invalid();
|
|
2243
|
+
return { attachment: created, upload: presigned(value.upload) };
|
|
2244
|
+
},
|
|
2245
|
+
async commit(attachmentId, owner, signal) {
|
|
2246
|
+
if (!attachmentIdPattern.test(attachmentId) || !object(owner) || owner.kind !== "message" && owner.kind !== "summary" || !id(owner.id))
|
|
2247
|
+
invalid();
|
|
2248
|
+
const value = await request(
|
|
2249
|
+
`/attachments/${attachmentId}/commit`,
|
|
2250
|
+
{
|
|
2251
|
+
method: "POST",
|
|
2252
|
+
body: JSON.stringify({
|
|
2253
|
+
account_id: accountId,
|
|
2254
|
+
owner_kind: owner.kind,
|
|
2255
|
+
owner_id: owner.id
|
|
2256
|
+
})
|
|
2257
|
+
},
|
|
2258
|
+
signal
|
|
2259
|
+
);
|
|
2260
|
+
if (!object(value)) invalid();
|
|
2261
|
+
const committed = attachment(value.attachment);
|
|
2262
|
+
if (committed.id !== attachmentId || committed.state !== "committed" || committed.ownerKind !== owner.kind || committed.ownerId !== owner.id)
|
|
2263
|
+
invalid();
|
|
2264
|
+
return committed;
|
|
2265
|
+
},
|
|
2266
|
+
async get(attachmentId, signal) {
|
|
2267
|
+
if (!attachmentIdPattern.test(attachmentId)) invalid();
|
|
2268
|
+
let value;
|
|
2269
|
+
try {
|
|
2270
|
+
value = await request(`/attachments/${attachmentId}?account_id=${accountId}`, {}, signal);
|
|
2271
|
+
} catch (error) {
|
|
2272
|
+
if (error instanceof ChatStoreError && error.status === 404) return null;
|
|
2273
|
+
throw error;
|
|
2274
|
+
}
|
|
2275
|
+
if (!object(value)) invalid();
|
|
2276
|
+
const found = attachment(value.attachment);
|
|
2277
|
+
if (found.id !== attachmentId) invalid();
|
|
2278
|
+
return {
|
|
2279
|
+
attachment: found,
|
|
2280
|
+
...value.download !== void 0 ? { download: presigned(value.download) } : {}
|
|
2281
|
+
};
|
|
2282
|
+
},
|
|
2283
|
+
async delete(attachmentId, signal) {
|
|
2284
|
+
if (!attachmentIdPattern.test(attachmentId)) invalid();
|
|
2285
|
+
await request(
|
|
2286
|
+
`/attachments/${attachmentId}?account_id=${accountId}`,
|
|
2287
|
+
{ method: "DELETE" },
|
|
2288
|
+
signal
|
|
2289
|
+
);
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
1969
2292
|
};
|
|
1970
2293
|
}
|
|
2294
|
+
var emptyMessage = {
|
|
2295
|
+
kind: "message",
|
|
2296
|
+
id: "query",
|
|
2297
|
+
conversationId: "query",
|
|
2298
|
+
sortKey: 0,
|
|
2299
|
+
createdAt: 1,
|
|
2300
|
+
updatedAt: 1,
|
|
2301
|
+
isDeleted: false,
|
|
2302
|
+
data: {}
|
|
2303
|
+
};
|
|
1971
2304
|
|
|
1972
2305
|
// src/lib/db/chat/lazyDecrypt.ts
|
|
1973
2306
|
var LRU_CAPACITY = 256;
|
|
@@ -2081,19 +2414,19 @@ function base64ToUint8Array(base64) {
|
|
|
2081
2414
|
// src/lib/memoryEngine/vectorEncoding.ts
|
|
2082
2415
|
var BYTES_PER_FLOAT32 = 4;
|
|
2083
2416
|
var CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
|
|
2084
|
-
function encodeChunkVector(
|
|
2085
|
-
const f32 =
|
|
2417
|
+
function encodeChunkVector(vector2) {
|
|
2418
|
+
const f32 = vector2 instanceof Float32Array ? vector2 : Float32Array.from(vector2);
|
|
2086
2419
|
return uint8ArrayToBase64(new Uint8Array(f32.buffer, f32.byteOffset, f32.byteLength));
|
|
2087
2420
|
}
|
|
2088
|
-
function decodeChunkVector(
|
|
2089
|
-
if (!
|
|
2090
|
-
if (typeof
|
|
2091
|
-
if (
|
|
2421
|
+
function decodeChunkVector(vector2, onMalformed) {
|
|
2422
|
+
if (!vector2 || vector2.length === 0) return new Float32Array(0);
|
|
2423
|
+
if (typeof vector2 !== "string") return Float32Array.from(vector2);
|
|
2424
|
+
if (vector2.length % 4 !== 0 || !CANONICAL_BASE64.test(vector2)) {
|
|
2092
2425
|
onMalformed?.();
|
|
2093
2426
|
return new Float32Array(0);
|
|
2094
2427
|
}
|
|
2095
2428
|
try {
|
|
2096
|
-
const bytes = base64ToUint8Array(
|
|
2429
|
+
const bytes = base64ToUint8Array(vector2);
|
|
2097
2430
|
if (bytes.byteLength === 0 || bytes.byteLength % BYTES_PER_FLOAT32 !== 0) {
|
|
2098
2431
|
onMalformed?.();
|
|
2099
2432
|
return new Float32Array(0);
|
|
@@ -2109,49 +2442,49 @@ function decodeChunkVector(vector, onMalformed) {
|
|
|
2109
2442
|
var DEFAULT_CHUNK_SIZE = 400;
|
|
2110
2443
|
var DEFAULT_CHUNK_OVERLAP = 50;
|
|
2111
2444
|
var DEFAULT_MIN_CHUNK_SIZE = 50;
|
|
2112
|
-
function splitIntoSentences(
|
|
2445
|
+
function splitIntoSentences(text3) {
|
|
2113
2446
|
const sentenceRegex = /[.!?]+[\s"')}\]]*(?=\s|$)/g;
|
|
2114
2447
|
const sentences = [];
|
|
2115
2448
|
let lastIndex = 0;
|
|
2116
2449
|
let match;
|
|
2117
|
-
while ((match = sentenceRegex.exec(
|
|
2450
|
+
while ((match = sentenceRegex.exec(text3)) !== null) {
|
|
2118
2451
|
const endIndex = match.index + match[0].length;
|
|
2119
|
-
const sentence =
|
|
2452
|
+
const sentence = text3.slice(lastIndex, endIndex).trim();
|
|
2120
2453
|
if (sentence) {
|
|
2121
2454
|
sentences.push(sentence);
|
|
2122
2455
|
}
|
|
2123
2456
|
lastIndex = endIndex;
|
|
2124
2457
|
}
|
|
2125
|
-
const remaining =
|
|
2458
|
+
const remaining = text3.slice(lastIndex).trim();
|
|
2126
2459
|
if (remaining) {
|
|
2127
2460
|
sentences.push(remaining);
|
|
2128
2461
|
}
|
|
2129
2462
|
return sentences;
|
|
2130
2463
|
}
|
|
2131
|
-
function chunkText(
|
|
2464
|
+
function chunkText(text3, options) {
|
|
2132
2465
|
const {
|
|
2133
2466
|
chunkSize = DEFAULT_CHUNK_SIZE,
|
|
2134
2467
|
chunkOverlap: requestedOverlap = DEFAULT_CHUNK_OVERLAP,
|
|
2135
2468
|
minChunkSize = DEFAULT_MIN_CHUNK_SIZE
|
|
2136
2469
|
} = options ?? {};
|
|
2137
2470
|
const chunkOverlap = Math.min(requestedOverlap, chunkSize - 1);
|
|
2138
|
-
if (
|
|
2471
|
+
if (text3.length <= chunkSize) {
|
|
2139
2472
|
return [
|
|
2140
2473
|
{
|
|
2141
|
-
text:
|
|
2474
|
+
text: text3.trim(),
|
|
2142
2475
|
startOffset: 0,
|
|
2143
|
-
endOffset:
|
|
2476
|
+
endOffset: text3.length
|
|
2144
2477
|
}
|
|
2145
2478
|
];
|
|
2146
2479
|
}
|
|
2147
|
-
const sentences = splitIntoSentences(
|
|
2480
|
+
const sentences = splitIntoSentences(text3);
|
|
2148
2481
|
const chunks = [];
|
|
2149
2482
|
let currentChunkSentences = [];
|
|
2150
2483
|
let currentChunkStart = 0;
|
|
2151
2484
|
let currentOffset = 0;
|
|
2152
2485
|
for (let i = 0; i < sentences.length; i++) {
|
|
2153
2486
|
const sentence = sentences[i];
|
|
2154
|
-
let sentenceStart =
|
|
2487
|
+
let sentenceStart = text3.indexOf(sentence, currentOffset);
|
|
2155
2488
|
if (sentenceStart === -1) {
|
|
2156
2489
|
sentenceStart = currentOffset;
|
|
2157
2490
|
}
|
|
@@ -2198,7 +2531,7 @@ function chunkText(text2, options) {
|
|
|
2198
2531
|
if (overlapSentences.length > 0) {
|
|
2199
2532
|
currentChunkSentences = [...overlapSentences, sentence];
|
|
2200
2533
|
const overlapText = overlapSentences[0];
|
|
2201
|
-
const overlapStart =
|
|
2534
|
+
const overlapStart = text3.lastIndexOf(overlapText, currentOffset);
|
|
2202
2535
|
currentChunkStart = overlapStart === -1 ? sentenceStart : overlapStart;
|
|
2203
2536
|
} else {
|
|
2204
2537
|
currentChunkSentences = [sentence];
|
|
@@ -2218,18 +2551,18 @@ function chunkText(text2, options) {
|
|
|
2218
2551
|
chunks.push({
|
|
2219
2552
|
text: chunkText2,
|
|
2220
2553
|
startOffset: currentChunkStart,
|
|
2221
|
-
endOffset:
|
|
2554
|
+
endOffset: text3.length
|
|
2222
2555
|
});
|
|
2223
2556
|
}
|
|
2224
2557
|
}
|
|
2225
2558
|
if (chunks.length === 0) {
|
|
2226
|
-
const trimmedText =
|
|
2559
|
+
const trimmedText = text3.trim();
|
|
2227
2560
|
if (trimmedText.length > 0) {
|
|
2228
2561
|
return [
|
|
2229
2562
|
{
|
|
2230
2563
|
text: trimmedText,
|
|
2231
2564
|
startOffset: 0,
|
|
2232
|
-
endOffset:
|
|
2565
|
+
endOffset: text3.length
|
|
2233
2566
|
}
|
|
2234
2567
|
];
|
|
2235
2568
|
}
|
|
@@ -2342,22 +2675,22 @@ function embeddingErrorFrom(response) {
|
|
|
2342
2675
|
const message = typeof response.error === "object" && response.error && "error" in response.error ? response.error.error : "API embedding failed";
|
|
2343
2676
|
return new EmbeddingHttpError(message, response.response?.status);
|
|
2344
2677
|
}
|
|
2345
|
-
async function generateEmbedding(
|
|
2678
|
+
async function generateEmbedding(text3, options) {
|
|
2346
2679
|
const total = options.totalTimeoutMs;
|
|
2347
|
-
if (total === void 0) return embedOne(
|
|
2348
|
-
return withDeadline((signal) => embedOne(
|
|
2680
|
+
if (total === void 0) return embedOne(text3, options, void 0);
|
|
2681
|
+
return withDeadline((signal) => embedOne(text3, options, signal), total, "embedding");
|
|
2349
2682
|
}
|
|
2350
2683
|
var { inFlightByCache, cachedEmbeddingModels } = getEmbeddingCacheState();
|
|
2351
|
-
async function embedOne(
|
|
2684
|
+
async function embedOne(text3, options, outer) {
|
|
2352
2685
|
const { cache } = options;
|
|
2353
|
-
if (!cache) return Array.from(await requestEmbedding(
|
|
2686
|
+
if (!cache) return Array.from(await requestEmbedding(text3, options, outer));
|
|
2354
2687
|
const model = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
2355
|
-
const cached = cache.get(
|
|
2688
|
+
const cached = cache.get(text3);
|
|
2356
2689
|
const cachedModel = cached && cachedEmbeddingModels.get(cached);
|
|
2357
2690
|
if (cached && (cachedModel === void 0 || cachedModel === model)) return Array.from(cached);
|
|
2358
2691
|
const inFlight = inFlightByCache.get(cache) ?? /* @__PURE__ */ new Map();
|
|
2359
2692
|
inFlightByCache.set(cache, inFlight);
|
|
2360
|
-
const requestKey = JSON.stringify([model,
|
|
2693
|
+
const requestKey = JSON.stringify([model, text3]);
|
|
2361
2694
|
const pending = inFlight.get(requestKey);
|
|
2362
2695
|
if (pending) {
|
|
2363
2696
|
try {
|
|
@@ -2366,18 +2699,18 @@ async function embedOne(text2, options, outer) {
|
|
|
2366
2699
|
if (outer?.aborted) throw err;
|
|
2367
2700
|
}
|
|
2368
2701
|
}
|
|
2369
|
-
const request = requestEmbedding(
|
|
2702
|
+
const request = requestEmbedding(text3, options, outer);
|
|
2370
2703
|
inFlight.set(requestKey, request);
|
|
2371
2704
|
try {
|
|
2372
2705
|
const embedding = await request;
|
|
2373
2706
|
cachedEmbeddingModels.set(embedding, model);
|
|
2374
|
-
cache.set(
|
|
2707
|
+
cache.set(text3, embedding);
|
|
2375
2708
|
return Array.from(embedding);
|
|
2376
2709
|
} finally {
|
|
2377
2710
|
if (inFlight.get(requestKey) === request) inFlight.delete(requestKey);
|
|
2378
2711
|
}
|
|
2379
2712
|
}
|
|
2380
|
-
async function requestEmbedding(
|
|
2713
|
+
async function requestEmbedding(text3, options, outer) {
|
|
2381
2714
|
const { baseUrl = BASE_URL, model } = options;
|
|
2382
2715
|
const timeoutMs = options.timeoutMs ?? DEFAULT_EMBEDDING_REQUEST_TIMEOUT_MS;
|
|
2383
2716
|
const headers = {
|
|
@@ -2389,7 +2722,7 @@ async function requestEmbedding(text2, options, outer) {
|
|
|
2389
2722
|
(signal) => postApiV1Embeddings({
|
|
2390
2723
|
baseUrl,
|
|
2391
2724
|
body: {
|
|
2392
|
-
input: options.maskInput ? options.maskInput(
|
|
2725
|
+
input: options.maskInput ? options.maskInput(text3) : text3,
|
|
2393
2726
|
model: model ?? DEFAULT_API_EMBEDDING_MODEL
|
|
2394
2727
|
},
|
|
2395
2728
|
headers,
|
|
@@ -2628,9 +2961,9 @@ function extractUserMessages(messages, maxCount, documentPattern) {
|
|
|
2628
2961
|
}
|
|
2629
2962
|
return result.reverse();
|
|
2630
2963
|
}
|
|
2631
|
-
function truncateAtWordBoundary(
|
|
2632
|
-
if (
|
|
2633
|
-
const truncated =
|
|
2964
|
+
function truncateAtWordBoundary(text3, maxLength) {
|
|
2965
|
+
if (text3.length <= maxLength) return text3;
|
|
2966
|
+
const truncated = text3.slice(0, maxLength);
|
|
2634
2967
|
const lastSpace = truncated.lastIndexOf(" ");
|
|
2635
2968
|
return lastSpace > maxLength * 0.5 ? truncated.slice(0, lastSpace) : truncated;
|
|
2636
2969
|
}
|
|
@@ -2755,12 +3088,12 @@ function buildSystemPrompt(ctx, options) {
|
|
|
2755
3088
|
let prompt = "";
|
|
2756
3089
|
const activeSections = [];
|
|
2757
3090
|
for (const section of sections) {
|
|
2758
|
-
const
|
|
2759
|
-
if (
|
|
2760
|
-
if (maxLength > 0 && prompt.length +
|
|
3091
|
+
const text3 = section.render(ctx);
|
|
3092
|
+
if (text3 === null) continue;
|
|
3093
|
+
if (maxLength > 0 && prompt.length + text3.length > maxLength) {
|
|
2761
3094
|
continue;
|
|
2762
3095
|
}
|
|
2763
|
-
prompt +=
|
|
3096
|
+
prompt += text3;
|
|
2764
3097
|
activeSections.push(section.key);
|
|
2765
3098
|
}
|
|
2766
3099
|
return { prompt, activeSections };
|
|
@@ -2967,15 +3300,15 @@ var PiiRedactor = class {
|
|
|
2967
3300
|
* (sequential, per-pattern) and the async {@link collectRegexSpans} (span)
|
|
2968
3301
|
* paths so the detection rules live in exactly one place.
|
|
2969
3302
|
*/
|
|
2970
|
-
findMatches(
|
|
3303
|
+
findMatches(text3, pattern) {
|
|
2971
3304
|
const regex = new RegExp(pattern.regex.source, pattern.regex.flags);
|
|
2972
3305
|
const found = [];
|
|
2973
3306
|
let m;
|
|
2974
|
-
while ((m = regex.exec(
|
|
3307
|
+
while ((m = regex.exec(text3)) !== null) {
|
|
2975
3308
|
const value = m[0];
|
|
2976
3309
|
if (pattern.validate && !pattern.validate(value)) continue;
|
|
2977
3310
|
if (pattern.context) {
|
|
2978
|
-
const before =
|
|
3311
|
+
const before = text3.slice(Math.max(0, m.index - 40), m.index);
|
|
2979
3312
|
if (!pattern.context.test(before)) continue;
|
|
2980
3313
|
}
|
|
2981
3314
|
found.push({ match: value, index: m.index });
|
|
@@ -2991,9 +3324,9 @@ var PiiRedactor = class {
|
|
|
2991
3324
|
* earlier patterns, so an earlier (more specific) pattern claims a region
|
|
2992
3325
|
* before a later one can — placeholders are inert and match no pattern.
|
|
2993
3326
|
*/
|
|
2994
|
-
scan(
|
|
3327
|
+
scan(text3, makeReplacement) {
|
|
2995
3328
|
const matches = [];
|
|
2996
|
-
let redacted =
|
|
3329
|
+
let redacted = text3;
|
|
2997
3330
|
for (const pattern of this.patterns) {
|
|
2998
3331
|
const found = this.findMatches(redacted, pattern);
|
|
2999
3332
|
if (found.length === 0) continue;
|
|
@@ -3014,8 +3347,8 @@ var PiiRedactor = class {
|
|
|
3014
3347
|
* Scan and redact PII from a single text string, using numbered, reversible
|
|
3015
3348
|
* placeholders ([EMAIL_1], …) tracked on this instance.
|
|
3016
3349
|
*/
|
|
3017
|
-
redactText(
|
|
3018
|
-
return this.scan(
|
|
3350
|
+
redactText(text3) {
|
|
3351
|
+
return this.scan(text3, (category, value) => this.getPlaceholder(category, value));
|
|
3019
3352
|
}
|
|
3020
3353
|
/**
|
|
3021
3354
|
* Mask PII with unnumbered, NON-reversible tokens ([EMAIL], [SSN], …) without
|
|
@@ -3024,15 +3357,15 @@ var PiiRedactor = class {
|
|
|
3024
3357
|
* embeddings endpoint while staying deterministic, so the same content always
|
|
3025
3358
|
* masks identically and search stays consistent across conversations.
|
|
3026
3359
|
*/
|
|
3027
|
-
maskText(
|
|
3028
|
-
return this.scan(
|
|
3360
|
+
maskText(text3) {
|
|
3361
|
+
return this.scan(text3, (category) => `[${category}]`).text;
|
|
3029
3362
|
}
|
|
3030
3363
|
/** All regex matches as spans on the ORIGINAL text (priority 0). Uses the
|
|
3031
3364
|
* same {@link findMatches} guards as the sync scan. */
|
|
3032
|
-
collectRegexSpans(
|
|
3365
|
+
collectRegexSpans(text3) {
|
|
3033
3366
|
const spans = [];
|
|
3034
3367
|
this.patterns.forEach((pattern, patternIndex) => {
|
|
3035
|
-
for (const { match, index } of this.findMatches(
|
|
3368
|
+
for (const { match, index } of this.findMatches(text3, pattern)) {
|
|
3036
3369
|
spans.push({
|
|
3037
3370
|
start: index,
|
|
3038
3371
|
end: index + match.length,
|
|
@@ -3051,16 +3384,16 @@ var PiiRedactor = class {
|
|
|
3051
3384
|
* of "Stripe") from producing a corrupting partial placeholder like
|
|
3052
3385
|
* `[ORG_1]e`.
|
|
3053
3386
|
*/
|
|
3054
|
-
normalizeNerSpans(
|
|
3387
|
+
normalizeNerSpans(text3, raw) {
|
|
3055
3388
|
if (!Array.isArray(raw)) return [];
|
|
3056
3389
|
const out = [];
|
|
3057
3390
|
raw.forEach((s, i) => {
|
|
3058
3391
|
if (!Number.isFinite(s.start) || !Number.isFinite(s.end)) return;
|
|
3059
|
-
let start = Math.max(0, Math.min(Math.trunc(s.start),
|
|
3060
|
-
let end = Math.max(0, Math.min(Math.trunc(s.end),
|
|
3392
|
+
let start = Math.max(0, Math.min(Math.trunc(s.start), text3.length));
|
|
3393
|
+
let end = Math.max(0, Math.min(Math.trunc(s.end), text3.length));
|
|
3061
3394
|
if (end <= start) return;
|
|
3062
|
-
while (start > 0 && isWordChar(
|
|
3063
|
-
while (end <
|
|
3395
|
+
while (start > 0 && isWordChar(text3[start - 1])) start--;
|
|
3396
|
+
while (end < text3.length && isWordChar(text3[end])) end++;
|
|
3064
3397
|
const category = String(s.category ?? "").trim();
|
|
3065
3398
|
if (!category) return;
|
|
3066
3399
|
out.push({ start, end, category, priority: 1, order: i });
|
|
@@ -3069,10 +3402,10 @@ var PiiRedactor = class {
|
|
|
3069
3402
|
}
|
|
3070
3403
|
/** Run the configured detector defensively: failures degrade to regex-only
|
|
3071
3404
|
* (fail-open on detection, not on protection) with a one-time warning. */
|
|
3072
|
-
async detectNerSpans(
|
|
3405
|
+
async detectNerSpans(text3) {
|
|
3073
3406
|
if (!this.nerDetector) return [];
|
|
3074
3407
|
try {
|
|
3075
|
-
return this.normalizeNerSpans(
|
|
3408
|
+
return this.normalizeNerSpans(text3, await this.nerDetector.detect(text3));
|
|
3076
3409
|
} catch (err) {
|
|
3077
3410
|
this.warnNerFailure(err);
|
|
3078
3411
|
return [];
|
|
@@ -3105,11 +3438,11 @@ var PiiRedactor = class {
|
|
|
3105
3438
|
* in pattern then document order, matching the sync scan's numbering). The
|
|
3106
3439
|
* string is then EMITTED in document order.
|
|
3107
3440
|
*/
|
|
3108
|
-
rebuildSpans(
|
|
3441
|
+
rebuildSpans(text3, spans, makeReplacement) {
|
|
3109
3442
|
const placeholderByStart = /* @__PURE__ */ new Map();
|
|
3110
3443
|
const matches = [];
|
|
3111
3444
|
for (const s of spans) {
|
|
3112
|
-
const value =
|
|
3445
|
+
const value = text3.slice(s.start, s.end);
|
|
3113
3446
|
const placeholder = makeReplacement(s.category, value);
|
|
3114
3447
|
placeholderByStart.set(s.start, placeholder);
|
|
3115
3448
|
matches.push({ category: s.category, original: value, placeholder });
|
|
@@ -3118,16 +3451,16 @@ var PiiRedactor = class {
|
|
|
3118
3451
|
let result = "";
|
|
3119
3452
|
let cursor = 0;
|
|
3120
3453
|
for (const s of emitOrder) {
|
|
3121
|
-
result +=
|
|
3454
|
+
result += text3.slice(cursor, s.start) + (placeholderByStart.get(s.start) ?? "");
|
|
3122
3455
|
cursor = s.end;
|
|
3123
3456
|
}
|
|
3124
|
-
result +=
|
|
3457
|
+
result += text3.slice(cursor);
|
|
3125
3458
|
return { text: result, matches };
|
|
3126
3459
|
}
|
|
3127
3460
|
/** Merge regex + NER spans for `text` into a resolved, non-overlapping set. */
|
|
3128
|
-
async detectAllSpans(
|
|
3129
|
-
const nerPromise = this.detectNerSpans(
|
|
3130
|
-
const regexSpans = this.collectRegexSpans(
|
|
3461
|
+
async detectAllSpans(text3) {
|
|
3462
|
+
const nerPromise = this.detectNerSpans(text3);
|
|
3463
|
+
const regexSpans = this.collectRegexSpans(text3);
|
|
3131
3464
|
const ner = await nerPromise;
|
|
3132
3465
|
return this.resolveOverlaps([...regexSpans, ...ner]);
|
|
3133
3466
|
}
|
|
@@ -3136,11 +3469,11 @@ var PiiRedactor = class {
|
|
|
3136
3469
|
* unstructured PII (the configured {@link NerDetector}) with numbered,
|
|
3137
3470
|
* reversible placeholders. With no detector, identical to `redactText`.
|
|
3138
3471
|
*/
|
|
3139
|
-
async redactTextAsync(
|
|
3140
|
-
if (!this.nerDetector) return this.redactText(
|
|
3141
|
-
const spans = await this.detectAllSpans(
|
|
3472
|
+
async redactTextAsync(text3) {
|
|
3473
|
+
if (!this.nerDetector) return this.redactText(text3);
|
|
3474
|
+
const spans = await this.detectAllSpans(text3);
|
|
3142
3475
|
return this.rebuildSpans(
|
|
3143
|
-
|
|
3476
|
+
text3,
|
|
3144
3477
|
spans,
|
|
3145
3478
|
(category, value) => this.getPlaceholder(category, value)
|
|
3146
3479
|
);
|
|
@@ -3149,10 +3482,10 @@ var PiiRedactor = class {
|
|
|
3149
3482
|
* Async counterpart to {@link maskText}: stateless `[CATEGORY]` masking over
|
|
3150
3483
|
* regex + NER matches. With no detector, identical to `maskText`.
|
|
3151
3484
|
*/
|
|
3152
|
-
async maskTextAsync(
|
|
3153
|
-
if (!this.nerDetector) return this.maskText(
|
|
3154
|
-
const spans = await this.detectAllSpans(
|
|
3155
|
-
return this.rebuildSpans(
|
|
3485
|
+
async maskTextAsync(text3) {
|
|
3486
|
+
if (!this.nerDetector) return this.maskText(text3);
|
|
3487
|
+
const spans = await this.detectAllSpans(text3);
|
|
3488
|
+
return this.rebuildSpans(text3, spans, (category) => `[${category}]`).text;
|
|
3156
3489
|
}
|
|
3157
3490
|
/**
|
|
3158
3491
|
* Async counterpart to {@link redactMessages}. Text parts are redacted
|
|
@@ -3238,9 +3571,9 @@ var PiiRedactor = class {
|
|
|
3238
3571
|
* {@link restoreForStorage} instead, which tolerates the mangled echoes the
|
|
3239
3572
|
* extraction models produce.
|
|
3240
3573
|
*/
|
|
3241
|
-
deAnonymize(
|
|
3242
|
-
if (!
|
|
3243
|
-
let restored =
|
|
3574
|
+
deAnonymize(text3) {
|
|
3575
|
+
if (!text3.includes("[")) return text3;
|
|
3576
|
+
let restored = text3;
|
|
3244
3577
|
for (const [placeholder, original] of this.placeholderToValue) {
|
|
3245
3578
|
if (restored.includes(placeholder)) {
|
|
3246
3579
|
restored = restored.split(placeholder).join(original);
|
|
@@ -3265,15 +3598,15 @@ var PiiRedactor = class {
|
|
|
3265
3598
|
* - `unresolved` is derived from the lookup miss during this pass, not from a
|
|
3266
3599
|
* regex re-scan of the output — so it can't trip over a restored value.
|
|
3267
3600
|
*/
|
|
3268
|
-
restoreForStorage(
|
|
3601
|
+
restoreForStorage(text3) {
|
|
3269
3602
|
if (this.storagePattern === void 0) this.storagePattern = this.buildStoragePattern();
|
|
3270
3603
|
if (!this.storagePattern || this.placeholderToValue.size === 0) {
|
|
3271
|
-
return { text:
|
|
3604
|
+
return { text: text3, unresolved: false };
|
|
3272
3605
|
}
|
|
3273
3606
|
this.storageLookup ?? (this.storageLookup = this.buildStorageLookup());
|
|
3274
3607
|
const { exact, ci, ciCollisions } = this.storageLookup;
|
|
3275
3608
|
let unresolved = false;
|
|
3276
|
-
const restored =
|
|
3609
|
+
const restored = text3.replace(
|
|
3277
3610
|
this.storagePattern,
|
|
3278
3611
|
(match, bracketed, bare) => {
|
|
3279
3612
|
const body = bracketed ?? bare ?? "";
|
|
@@ -3769,7 +4102,7 @@ function findMatchingTools(promptEmbeddings, tools, options) {
|
|
|
3769
4102
|
if (!tools || tools.length === 0) {
|
|
3770
4103
|
return [];
|
|
3771
4104
|
}
|
|
3772
|
-
const
|
|
4105
|
+
const embeddings2 = Array.isArray(promptEmbeddings[0]) ? promptEmbeddings : [promptEmbeddings];
|
|
3773
4106
|
const results = [];
|
|
3774
4107
|
for (const tool of tools) {
|
|
3775
4108
|
if (!tool.embedding || tool.embedding.length === 0) {
|
|
@@ -3777,7 +4110,7 @@ function findMatchingTools(promptEmbeddings, tools, options) {
|
|
|
3777
4110
|
}
|
|
3778
4111
|
try {
|
|
3779
4112
|
let maxSimilarity = -Infinity;
|
|
3780
|
-
for (const embedding of
|
|
4113
|
+
for (const embedding of embeddings2) {
|
|
3781
4114
|
const similarity = cosineSimilarity(embedding, tool.embedding);
|
|
3782
4115
|
if (similarity > maxSimilarity) {
|
|
3783
4116
|
maxSimilarity = similarity;
|
|
@@ -3808,12 +4141,12 @@ function scoreTools(promptEmbeddings, tools) {
|
|
|
3808
4141
|
const scores = /* @__PURE__ */ new Map();
|
|
3809
4142
|
if (!promptEmbeddings || promptEmbeddings.length === 0) return scores;
|
|
3810
4143
|
if (!tools || tools.length === 0) return scores;
|
|
3811
|
-
const
|
|
4144
|
+
const embeddings2 = Array.isArray(promptEmbeddings[0]) ? promptEmbeddings : [promptEmbeddings];
|
|
3812
4145
|
for (const tool of tools) {
|
|
3813
4146
|
if (!tool.embedding || tool.embedding.length === 0) continue;
|
|
3814
4147
|
let maxSimilarity = -Infinity;
|
|
3815
4148
|
try {
|
|
3816
|
-
for (const embedding of
|
|
4149
|
+
for (const embedding of embeddings2) {
|
|
3817
4150
|
const similarity = cosineSimilarity(embedding, tool.embedding);
|
|
3818
4151
|
if (similarity > maxSimilarity) maxSimilarity = similarity;
|
|
3819
4152
|
}
|
|
@@ -4035,13 +4368,13 @@ function createServerToolsFilter(options = {}) {
|
|
|
4035
4368
|
const exclude = new Set(options.excludeTools ?? []);
|
|
4036
4369
|
const sets = options.toolSets ?? [];
|
|
4037
4370
|
const matchOpts = options.matchOptions;
|
|
4038
|
-
const filter = (
|
|
4039
|
-
const matches = findMatchingTools(
|
|
4371
|
+
const filter = (embeddings2, tools) => {
|
|
4372
|
+
const matches = findMatchingTools(embeddings2, tools, matchOpts);
|
|
4040
4373
|
if (matches.length === 0) return [];
|
|
4041
4374
|
const matchedNames = new Set(matches.map((m) => m.tool.name));
|
|
4042
4375
|
let finalNames;
|
|
4043
4376
|
if (sets.length > 0) {
|
|
4044
|
-
const scores = scoreTools(
|
|
4377
|
+
const scores = scoreTools(embeddings2, tools);
|
|
4045
4378
|
const selectedScores = new Map(
|
|
4046
4379
|
[...scores].filter(([name]) => matchedNames.has(name) && !exclude.has(name))
|
|
4047
4380
|
);
|