@ontos-ai/knowhere-sdk 2.0.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -21
- package/dist/index.d.mts +403 -176
- package/dist/index.d.ts +403 -176
- package/dist/index.js +1970 -51
- package/dist/index.mjs +1966 -50
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -309,8 +309,8 @@ function enrichParseResult(parseResult2, scope) {
|
|
|
309
309
|
}
|
|
310
310
|
return parseResult2;
|
|
311
311
|
}
|
|
312
|
-
function sanitizePath(
|
|
313
|
-
let sanitized =
|
|
312
|
+
function sanitizePath(path4) {
|
|
313
|
+
let sanitized = path4.replace(/^\/+/, "");
|
|
314
314
|
sanitized = sanitized.replace(/\.\.(\/|\\)/g, "");
|
|
315
315
|
sanitized = sanitized.replace(/\\/g, "/");
|
|
316
316
|
return sanitized;
|
|
@@ -696,8 +696,8 @@ var BaseResource = class {
|
|
|
696
696
|
constructor(httpClient) {
|
|
697
697
|
this.httpClient = httpClient;
|
|
698
698
|
}
|
|
699
|
-
endpoint(
|
|
700
|
-
return `/v2${
|
|
699
|
+
endpoint(path4) {
|
|
700
|
+
return `/v2${path4.startsWith("/") ? path4 : `/${path4}`}`;
|
|
701
701
|
}
|
|
702
702
|
};
|
|
703
703
|
|
|
@@ -913,17 +913,28 @@ async function saveExpandedParseResult(result, directory) {
|
|
|
913
913
|
if (result.rawZip.length > 0) {
|
|
914
914
|
const didExtractZip = await tryExtractRawZip(result.rawZip, directory);
|
|
915
915
|
if (didExtractZip) {
|
|
916
|
+
await writeExpandedParseResultMetadata(result, directory);
|
|
916
917
|
return directory;
|
|
917
918
|
}
|
|
918
919
|
}
|
|
919
920
|
await fs2.mkdir(directory, { recursive: true });
|
|
921
|
+
await writeExpandedParseResultMetadata(result, directory);
|
|
922
|
+
for (const imageChunk of result.imageChunks) {
|
|
923
|
+
await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data);
|
|
924
|
+
}
|
|
925
|
+
for (const tableChunk of result.tableChunks) {
|
|
926
|
+
await writeTextAsset(directory, tableChunk.filePath, tableChunk.html);
|
|
927
|
+
}
|
|
928
|
+
return directory;
|
|
929
|
+
}
|
|
930
|
+
async function writeExpandedParseResultMetadata(result, directory) {
|
|
920
931
|
await fs2.writeFile(join(directory, "manifest.json"), JSON.stringify(result.manifest, null, 2));
|
|
921
932
|
if (result.docNav) {
|
|
922
933
|
await fs2.writeFile(join(directory, "doc_nav.json"), JSON.stringify(result.docNav, null, 2));
|
|
923
934
|
}
|
|
924
935
|
await fs2.writeFile(
|
|
925
936
|
join(directory, "chunks.json"),
|
|
926
|
-
JSON.stringify(serializeChunks(result.chunks), null, 2)
|
|
937
|
+
JSON.stringify(keysToSnake(serializeChunks(result.chunks)), null, 2)
|
|
927
938
|
);
|
|
928
939
|
if (result.chunksSlim) {
|
|
929
940
|
await fs2.writeFile(
|
|
@@ -952,13 +963,6 @@ async function saveExpandedParseResult(result, directory) {
|
|
|
952
963
|
if (result.hierarchyViewHtml) {
|
|
953
964
|
await fs2.writeFile(join(directory, "hierarchy_view.html"), result.hierarchyViewHtml);
|
|
954
965
|
}
|
|
955
|
-
for (const imageChunk of result.imageChunks) {
|
|
956
|
-
await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data);
|
|
957
|
-
}
|
|
958
|
-
for (const tableChunk of result.tableChunks) {
|
|
959
|
-
await writeTextAsset(directory, tableChunk.filePath, tableChunk.html);
|
|
960
|
-
}
|
|
961
|
-
return directory;
|
|
962
966
|
}
|
|
963
967
|
async function tryExtractRawZip(zipBuffer, directory) {
|
|
964
968
|
try {
|
|
@@ -1097,13 +1101,14 @@ function buildTextChunk(chunkData) {
|
|
|
1097
1101
|
};
|
|
1098
1102
|
}
|
|
1099
1103
|
function buildPageChunk(chunkData) {
|
|
1104
|
+
const metadata = chunkData.metadata ?? {};
|
|
1100
1105
|
return {
|
|
1101
1106
|
chunkId: chunkData.chunkId ?? "",
|
|
1102
1107
|
type: "page",
|
|
1103
1108
|
contentSource: chunkData.contentSource ?? "summary",
|
|
1104
1109
|
content: chunkData.content ?? "",
|
|
1105
1110
|
path: chunkData.path ?? "",
|
|
1106
|
-
metadata
|
|
1111
|
+
metadata
|
|
1107
1112
|
};
|
|
1108
1113
|
}
|
|
1109
1114
|
function buildImageChunk(chunkData, filePath, imageBuffer) {
|
|
@@ -1114,6 +1119,7 @@ function buildImageChunk(chunkData, filePath, imageBuffer) {
|
|
|
1114
1119
|
content: chunkData.content ?? "",
|
|
1115
1120
|
path: chunkData.path ?? "",
|
|
1116
1121
|
filePath,
|
|
1122
|
+
assetUrl: chunkData.assetUrl,
|
|
1117
1123
|
data: imageBuffer,
|
|
1118
1124
|
metadata: chunkData.metadata ?? {},
|
|
1119
1125
|
get format() {
|
|
@@ -1132,6 +1138,7 @@ function buildTableChunk(chunkData, filePath, html) {
|
|
|
1132
1138
|
content: chunkData.content ?? "",
|
|
1133
1139
|
path: chunkData.path ?? "",
|
|
1134
1140
|
filePath,
|
|
1141
|
+
assetUrl: chunkData.assetUrl,
|
|
1135
1142
|
html,
|
|
1136
1143
|
metadata: chunkData.metadata ?? {},
|
|
1137
1144
|
async save(directory) {
|
|
@@ -1226,6 +1233,7 @@ function serializeChunks(chunks) {
|
|
|
1226
1233
|
};
|
|
1227
1234
|
if (chunk.type === "image" || chunk.type === "table") {
|
|
1228
1235
|
rawChunk.filePath = chunk.filePath;
|
|
1236
|
+
rawChunk.assetUrl = chunk.assetUrl;
|
|
1229
1237
|
}
|
|
1230
1238
|
return rawChunk;
|
|
1231
1239
|
})
|
|
@@ -1434,6 +1442,14 @@ var Documents = class extends BaseResource {
|
|
|
1434
1442
|
this.createChunkGetRequestConfig(params)
|
|
1435
1443
|
);
|
|
1436
1444
|
}
|
|
1445
|
+
/**
|
|
1446
|
+
* Get a signed normalized source-file URL for compatibility and debugging.
|
|
1447
|
+
*/
|
|
1448
|
+
async getPageCitationSource(documentId) {
|
|
1449
|
+
return this.httpClient.get(
|
|
1450
|
+
this.endpoint(`/documents/${documentId}/files/page-citation-source`)
|
|
1451
|
+
);
|
|
1452
|
+
}
|
|
1437
1453
|
/**
|
|
1438
1454
|
* Archive one canonical document by ID.
|
|
1439
1455
|
*/
|
|
@@ -1506,6 +1522,477 @@ var Documents = class extends BaseResource {
|
|
|
1506
1522
|
// src/knowledge/knowledge.ts
|
|
1507
1523
|
import { createHash as createHash2 } from "crypto";
|
|
1508
1524
|
|
|
1525
|
+
// src/storage/asset-storage.ts
|
|
1526
|
+
import JSZip2 from "jszip";
|
|
1527
|
+
var pageCitationAssetsMetadataKey = "pageAssets";
|
|
1528
|
+
var defaultImageContentType = "application/octet-stream";
|
|
1529
|
+
var tableContentType = "text/html; charset=utf-8";
|
|
1530
|
+
async function storeParseResultAssets(result, options) {
|
|
1531
|
+
if (!options) {
|
|
1532
|
+
return {
|
|
1533
|
+
result,
|
|
1534
|
+
assetUrlsByFilePath: {}
|
|
1535
|
+
};
|
|
1536
|
+
}
|
|
1537
|
+
const normalizedKeyPrefix = normalizeStorageKeyPrefix(options.keyPrefix);
|
|
1538
|
+
const assets = dedupeStorageAssets(await collectStorageAssets(result, normalizedKeyPrefix));
|
|
1539
|
+
const assetUrlsByFilePath = {};
|
|
1540
|
+
for (const asset of assets) {
|
|
1541
|
+
const storedUrl = await storeAsset(asset, options.adapter, options.skipExisting ?? true);
|
|
1542
|
+
if (storedUrl) {
|
|
1543
|
+
assetUrlsByFilePath[asset.sourcePath] = storedUrl;
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
const normalizedResult = Object.keys(assetUrlsByFilePath).length === 0 ? result : rewriteResultAssetUrls(result, assetUrlsByFilePath);
|
|
1547
|
+
return {
|
|
1548
|
+
result: normalizedResult,
|
|
1549
|
+
assetUrlsByFilePath
|
|
1550
|
+
};
|
|
1551
|
+
}
|
|
1552
|
+
async function collectStorageAssets(result, keyPrefix) {
|
|
1553
|
+
const resultZip = await loadResultZipForPageCitationAssets(result);
|
|
1554
|
+
return [
|
|
1555
|
+
...result.imageChunks.flatMap((chunk) => collectImageStorageAsset(chunk, keyPrefix)),
|
|
1556
|
+
...result.tableChunks.flatMap((chunk) => collectTableStorageAsset(chunk, keyPrefix)),
|
|
1557
|
+
...resultZip ? await collectPageCitationStorageAssets(result.pageChunks, keyPrefix, resultZip) : []
|
|
1558
|
+
];
|
|
1559
|
+
}
|
|
1560
|
+
function collectImageStorageAsset(chunk, keyPrefix) {
|
|
1561
|
+
const sourcePath = normalizeResultAssetPath(chunk.filePath);
|
|
1562
|
+
if (!sourcePath) return [];
|
|
1563
|
+
return [
|
|
1564
|
+
{
|
|
1565
|
+
sourcePath,
|
|
1566
|
+
storageKey: `${keyPrefix}/${sourcePath}`,
|
|
1567
|
+
body: chunk.data,
|
|
1568
|
+
contentType: getImageContentType(chunk.filePath),
|
|
1569
|
+
metadata: {
|
|
1570
|
+
chunkId: chunk.chunkId,
|
|
1571
|
+
chunkType: chunk.type,
|
|
1572
|
+
sourcePath
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
];
|
|
1576
|
+
}
|
|
1577
|
+
function collectTableStorageAsset(chunk, keyPrefix) {
|
|
1578
|
+
const sourcePath = normalizeResultAssetPath(chunk.filePath);
|
|
1579
|
+
if (!sourcePath) return [];
|
|
1580
|
+
return [
|
|
1581
|
+
{
|
|
1582
|
+
sourcePath,
|
|
1583
|
+
storageKey: `${keyPrefix}/${sourcePath}`,
|
|
1584
|
+
body: Buffer.from(chunk.html, "utf8"),
|
|
1585
|
+
contentType: tableContentType,
|
|
1586
|
+
metadata: {
|
|
1587
|
+
chunkId: chunk.chunkId,
|
|
1588
|
+
chunkType: chunk.type,
|
|
1589
|
+
sourcePath
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
];
|
|
1593
|
+
}
|
|
1594
|
+
async function collectPageCitationStorageAssets(chunks, keyPrefix, resultZip) {
|
|
1595
|
+
const assets = [];
|
|
1596
|
+
for (const chunk of chunks) {
|
|
1597
|
+
const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]);
|
|
1598
|
+
for (const pageAsset of pageAssets) {
|
|
1599
|
+
const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef);
|
|
1600
|
+
if (!sourcePath) continue;
|
|
1601
|
+
const zipEntry = resultZip.file(sourcePath);
|
|
1602
|
+
if (!zipEntry) continue;
|
|
1603
|
+
assets.push({
|
|
1604
|
+
sourcePath,
|
|
1605
|
+
storageKey: `${keyPrefix}/${sourcePath}`,
|
|
1606
|
+
body: await zipEntry.async("uint8array"),
|
|
1607
|
+
contentType: pageAsset.contentType,
|
|
1608
|
+
metadata: getPageCitationAssetMetadata(chunk, pageAsset, sourcePath)
|
|
1609
|
+
});
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
return assets;
|
|
1613
|
+
}
|
|
1614
|
+
function getPageCitationAssetMetadata(chunk, pageAsset, sourcePath) {
|
|
1615
|
+
return {
|
|
1616
|
+
chunkId: chunk.chunkId,
|
|
1617
|
+
chunkType: chunk.type,
|
|
1618
|
+
pageNum: String(pageAsset.pageNum),
|
|
1619
|
+
sourcePath
|
|
1620
|
+
};
|
|
1621
|
+
}
|
|
1622
|
+
async function loadResultZipForPageCitationAssets(result) {
|
|
1623
|
+
if (!hasPageCitationAssetReferences(result) || result.rawZip.length === 0) {
|
|
1624
|
+
return null;
|
|
1625
|
+
}
|
|
1626
|
+
try {
|
|
1627
|
+
return await JSZip2.loadAsync(result.rawZip);
|
|
1628
|
+
} catch (cause) {
|
|
1629
|
+
throw new Error("Unable to read page citation assets from the Knowhere result ZIP.", {
|
|
1630
|
+
cause
|
|
1631
|
+
});
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
function hasPageCitationAssetReferences(result) {
|
|
1635
|
+
return result.pageChunks.some(
|
|
1636
|
+
(chunk) => parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]).length > 0
|
|
1637
|
+
);
|
|
1638
|
+
}
|
|
1639
|
+
function dedupeStorageAssets(assets) {
|
|
1640
|
+
const assetsBySourcePath = /* @__PURE__ */ new Map();
|
|
1641
|
+
for (const asset of assets) {
|
|
1642
|
+
if (!assetsBySourcePath.has(asset.sourcePath)) {
|
|
1643
|
+
assetsBySourcePath.set(asset.sourcePath, asset);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
return [...assetsBySourcePath.values()];
|
|
1647
|
+
}
|
|
1648
|
+
async function storeAsset(asset, adapter, skipExisting) {
|
|
1649
|
+
if (skipExisting && adapter.headObject) {
|
|
1650
|
+
const existing = await adapter.headObject(asset.storageKey);
|
|
1651
|
+
if (existing) {
|
|
1652
|
+
const existingUrl = existing.url ?? await adapter.getObjectUrl?.(asset.storageKey);
|
|
1653
|
+
if (existingUrl) return existingUrl;
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
const writeResult = await adapter.writeObject({
|
|
1657
|
+
key: asset.storageKey,
|
|
1658
|
+
body: asset.body,
|
|
1659
|
+
contentType: asset.contentType,
|
|
1660
|
+
metadata: asset.metadata
|
|
1661
|
+
});
|
|
1662
|
+
return writeResult.url ?? await adapter.getObjectUrl?.(writeResult.key) ?? null;
|
|
1663
|
+
}
|
|
1664
|
+
function rewriteResultAssetUrls(result, assetUrlsByFilePath) {
|
|
1665
|
+
const chunks = result.chunks.map((chunk) => rewriteChunkAssetUrls(chunk, assetUrlsByFilePath));
|
|
1666
|
+
result.chunks.splice(0, result.chunks.length, ...chunks);
|
|
1667
|
+
refreshStaticChunkCollections(result);
|
|
1668
|
+
return result;
|
|
1669
|
+
}
|
|
1670
|
+
function rewriteChunkAssetUrls(chunk, assetUrlsByPath) {
|
|
1671
|
+
if (chunk.type === "image" || chunk.type === "table") {
|
|
1672
|
+
const sourcePath = normalizeResultAssetPath(chunk.filePath);
|
|
1673
|
+
const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : void 0;
|
|
1674
|
+
if (!assetUrl) return chunk;
|
|
1675
|
+
return {
|
|
1676
|
+
...chunk,
|
|
1677
|
+
assetUrl
|
|
1678
|
+
};
|
|
1679
|
+
}
|
|
1680
|
+
if (chunk.type === "page") {
|
|
1681
|
+
const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]);
|
|
1682
|
+
if (pageAssets.length === 0) return chunk;
|
|
1683
|
+
const rewrittenPageAssets = pageAssets.map((pageAsset) => {
|
|
1684
|
+
const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef);
|
|
1685
|
+
const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : void 0;
|
|
1686
|
+
return assetUrl ? { ...pageAsset, assetUrl } : pageAsset;
|
|
1687
|
+
});
|
|
1688
|
+
return {
|
|
1689
|
+
...chunk,
|
|
1690
|
+
metadata: {
|
|
1691
|
+
...chunk.metadata,
|
|
1692
|
+
[pageCitationAssetsMetadataKey]: rewrittenPageAssets
|
|
1693
|
+
}
|
|
1694
|
+
};
|
|
1695
|
+
}
|
|
1696
|
+
return chunk;
|
|
1697
|
+
}
|
|
1698
|
+
function parsePageCitationAssets(value) {
|
|
1699
|
+
if (!Array.isArray(value)) return [];
|
|
1700
|
+
return value.flatMap((item) => {
|
|
1701
|
+
if (!isRecord(item)) return [];
|
|
1702
|
+
const pageNum = item.pageNum;
|
|
1703
|
+
const artifactRef = item.artifactRef;
|
|
1704
|
+
const contentType = item.contentType;
|
|
1705
|
+
const source = item.source;
|
|
1706
|
+
if (typeof pageNum !== "number" || !Number.isSafeInteger(pageNum) || pageNum <= 0) return [];
|
|
1707
|
+
if (typeof artifactRef !== "string") return [];
|
|
1708
|
+
if (contentType !== "image/png" && contentType !== "image/jpeg") return [];
|
|
1709
|
+
if (source !== "knowhere-rendered-page-citation-source") return [];
|
|
1710
|
+
return [
|
|
1711
|
+
{
|
|
1712
|
+
pageNum,
|
|
1713
|
+
artifactRef,
|
|
1714
|
+
assetUrl: typeof item.assetUrl === "string" ? item.assetUrl : void 0,
|
|
1715
|
+
contentType,
|
|
1716
|
+
width: typeof item.width === "number" ? item.width : void 0,
|
|
1717
|
+
height: typeof item.height === "number" ? item.height : void 0,
|
|
1718
|
+
source
|
|
1719
|
+
}
|
|
1720
|
+
];
|
|
1721
|
+
});
|
|
1722
|
+
}
|
|
1723
|
+
function normalizeStorageKeyPrefix(value) {
|
|
1724
|
+
const normalized = normalizeStoragePath(value, { allowDotPrefix: false });
|
|
1725
|
+
if (!normalized) {
|
|
1726
|
+
throw new Error("storageAdapter keyPrefix must be a non-empty relative POSIX path.");
|
|
1727
|
+
}
|
|
1728
|
+
return normalized;
|
|
1729
|
+
}
|
|
1730
|
+
function normalizeResultAssetPath(value) {
|
|
1731
|
+
return normalizeStoragePath(value, { allowDotPrefix: true });
|
|
1732
|
+
}
|
|
1733
|
+
function normalizeStoragePath(value, options) {
|
|
1734
|
+
if (!value) return null;
|
|
1735
|
+
const trimmed = value.trim();
|
|
1736
|
+
if (trimmed.length === 0 || trimmed.includes("\0")) return null;
|
|
1737
|
+
if (trimmed.startsWith("/") || /^[A-Za-z]:/.test(trimmed)) return null;
|
|
1738
|
+
const normalized = options.allowDotPrefix ? trimmed.replaceAll("\\", "/").replace(/^\.\/+/, "") : trimmed.replaceAll("\\", "/");
|
|
1739
|
+
if (normalized.length === 0) return null;
|
|
1740
|
+
const parts = normalized.split("/");
|
|
1741
|
+
if (parts.some((part) => part.length === 0 || part === "." || part === "..")) return null;
|
|
1742
|
+
return parts.join("/");
|
|
1743
|
+
}
|
|
1744
|
+
function refreshStaticChunkCollections(result) {
|
|
1745
|
+
replaceStaticChunkCollection(
|
|
1746
|
+
result,
|
|
1747
|
+
"textChunks",
|
|
1748
|
+
result.chunks.filter((chunk) => chunk.type === "text")
|
|
1749
|
+
);
|
|
1750
|
+
replaceStaticChunkCollection(
|
|
1751
|
+
result,
|
|
1752
|
+
"imageChunks",
|
|
1753
|
+
result.chunks.filter((chunk) => chunk.type === "image")
|
|
1754
|
+
);
|
|
1755
|
+
replaceStaticChunkCollection(
|
|
1756
|
+
result,
|
|
1757
|
+
"tableChunks",
|
|
1758
|
+
result.chunks.filter((chunk) => chunk.type === "table")
|
|
1759
|
+
);
|
|
1760
|
+
replaceStaticChunkCollection(
|
|
1761
|
+
result,
|
|
1762
|
+
"pageChunks",
|
|
1763
|
+
result.chunks.filter((chunk) => chunk.type === "page")
|
|
1764
|
+
);
|
|
1765
|
+
}
|
|
1766
|
+
function replaceStaticChunkCollection(result, property, value) {
|
|
1767
|
+
const descriptor = Object.getOwnPropertyDescriptor(result, property);
|
|
1768
|
+
if (!descriptor || descriptor.get || descriptor.writable !== true) return;
|
|
1769
|
+
Object.defineProperty(result, property, {
|
|
1770
|
+
...descriptor,
|
|
1771
|
+
value
|
|
1772
|
+
});
|
|
1773
|
+
}
|
|
1774
|
+
function getImageContentType(filePath) {
|
|
1775
|
+
const extension = filePath.split(".").at(-1)?.toLowerCase();
|
|
1776
|
+
if (extension === "jpg" || extension === "jpeg") return "image/jpeg";
|
|
1777
|
+
if (extension === "png") return "image/png";
|
|
1778
|
+
if (extension === "webp") return "image/webp";
|
|
1779
|
+
if (extension === "gif") return "image/gif";
|
|
1780
|
+
if (extension === "svg") return "image/svg+xml";
|
|
1781
|
+
return defaultImageContentType;
|
|
1782
|
+
}
|
|
1783
|
+
function isRecord(value) {
|
|
1784
|
+
return typeof value === "object" && value !== null;
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
// src/storage/parsed-document-storage.ts
|
|
1788
|
+
import JSZip3 from "jszip";
|
|
1789
|
+
var parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
|
|
1790
|
+
var jsonContentType = "application/json; charset=utf-8";
|
|
1791
|
+
var markdownContentType = "text/markdown; charset=utf-8";
|
|
1792
|
+
var csvContentType = "text/csv; charset=utf-8";
|
|
1793
|
+
var htmlContentType = "text/html; charset=utf-8";
|
|
1794
|
+
var binaryContentType = "application/octet-stream";
|
|
1795
|
+
async function syncParseResultToParsedDocumentStorage(params) {
|
|
1796
|
+
const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
|
|
1797
|
+
const revisionKey = params.revisionKey ?? params.result.jobId;
|
|
1798
|
+
const objects = await collectResultLayoutObjects(params.result);
|
|
1799
|
+
for (const object of objects) {
|
|
1800
|
+
await params.storage.writeObject({
|
|
1801
|
+
documentId,
|
|
1802
|
+
revisionKey,
|
|
1803
|
+
...object
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
const commit = {
|
|
1807
|
+
version: 1,
|
|
1808
|
+
documentId,
|
|
1809
|
+
revisionKey,
|
|
1810
|
+
source: params.source ?? "resultZip",
|
|
1811
|
+
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1812
|
+
};
|
|
1813
|
+
await writeParsedDocumentJsonObject(params.storage, {
|
|
1814
|
+
documentId,
|
|
1815
|
+
revisionKey,
|
|
1816
|
+
path: parsedDocumentCommitPath,
|
|
1817
|
+
value: commit
|
|
1818
|
+
});
|
|
1819
|
+
return {
|
|
1820
|
+
result: params.result,
|
|
1821
|
+
documentId,
|
|
1822
|
+
revisionKey,
|
|
1823
|
+
commit
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1826
|
+
async function readParsedDocumentCommit(params) {
|
|
1827
|
+
const object = await params.storage.readObject({
|
|
1828
|
+
documentId: params.documentId,
|
|
1829
|
+
revisionKey: params.revisionKey,
|
|
1830
|
+
path: parsedDocumentCommitPath
|
|
1831
|
+
});
|
|
1832
|
+
if (!object) {
|
|
1833
|
+
return null;
|
|
1834
|
+
}
|
|
1835
|
+
const commit = readJsonObject(object);
|
|
1836
|
+
return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
|
|
1837
|
+
}
|
|
1838
|
+
function readJsonObject(object) {
|
|
1839
|
+
return JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
1840
|
+
}
|
|
1841
|
+
async function collectResultLayoutObjects(result) {
|
|
1842
|
+
const objectsByPath = /* @__PURE__ */ new Map();
|
|
1843
|
+
const rawZipObjects = await collectRawZipObjects(result.rawZip);
|
|
1844
|
+
for (const object of rawZipObjects) {
|
|
1845
|
+
objectsByPath.set(object.path, object);
|
|
1846
|
+
}
|
|
1847
|
+
for (const object of collectGeneratedResultObjects(result)) {
|
|
1848
|
+
if (!objectsByPath.has(object.path)) {
|
|
1849
|
+
objectsByPath.set(object.path, object);
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
return [...objectsByPath.values()].filter((object) => !isSdkMetadataPath(object.path));
|
|
1853
|
+
}
|
|
1854
|
+
async function collectRawZipObjects(rawZip) {
|
|
1855
|
+
try {
|
|
1856
|
+
const zip = await JSZip3.loadAsync(rawZip);
|
|
1857
|
+
const objects = [];
|
|
1858
|
+
for (const entry of Object.values(zip.files)) {
|
|
1859
|
+
if (entry.dir || entry.name === "result.zip") {
|
|
1860
|
+
continue;
|
|
1861
|
+
}
|
|
1862
|
+
objects.push({
|
|
1863
|
+
path: normalizeResultPath(entry.name),
|
|
1864
|
+
body: new Uint8Array(await entry.async("nodebuffer")),
|
|
1865
|
+
contentType: inferContentTypeFromPath(entry.name)
|
|
1866
|
+
});
|
|
1867
|
+
}
|
|
1868
|
+
return objects;
|
|
1869
|
+
} catch {
|
|
1870
|
+
return [];
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
function collectGeneratedResultObjects(result) {
|
|
1874
|
+
const objects = [
|
|
1875
|
+
createJsonObject("manifest.json", result.manifest),
|
|
1876
|
+
createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)))
|
|
1877
|
+
];
|
|
1878
|
+
if (result.docNav) {
|
|
1879
|
+
objects.push(createJsonObject("doc_nav.json", result.docNav));
|
|
1880
|
+
}
|
|
1881
|
+
if (result.fullMarkdown) {
|
|
1882
|
+
objects.push(createTextObject("full.md", result.fullMarkdown, markdownContentType));
|
|
1883
|
+
}
|
|
1884
|
+
if (result.chunksSlim) {
|
|
1885
|
+
objects.push(createJsonObject("chunks_slim.json", { chunks: result.chunksSlim }));
|
|
1886
|
+
}
|
|
1887
|
+
if (result.hierarchy) {
|
|
1888
|
+
objects.push(createJsonObject("hierarchy.json", result.hierarchy));
|
|
1889
|
+
}
|
|
1890
|
+
if (result.tocHierarchies) {
|
|
1891
|
+
objects.push(createJsonObject("toc_hierarchies.json", result.tocHierarchies));
|
|
1892
|
+
}
|
|
1893
|
+
if (result.kbCsv) {
|
|
1894
|
+
objects.push(createTextObject("kb.csv", result.kbCsv, csvContentType));
|
|
1895
|
+
}
|
|
1896
|
+
if (result.hierarchyViewHtml) {
|
|
1897
|
+
objects.push(
|
|
1898
|
+
createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType)
|
|
1899
|
+
);
|
|
1900
|
+
}
|
|
1901
|
+
for (const chunk of result.imageChunks) {
|
|
1902
|
+
objects.push({
|
|
1903
|
+
path: normalizeResultPath(chunk.filePath),
|
|
1904
|
+
body: new Uint8Array(chunk.data),
|
|
1905
|
+
contentType: inferContentTypeFromPath(chunk.filePath),
|
|
1906
|
+
metadata: {
|
|
1907
|
+
chunkId: chunk.chunkId,
|
|
1908
|
+
chunkType: chunk.type,
|
|
1909
|
+
sourcePath: chunk.filePath
|
|
1910
|
+
}
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
for (const chunk of result.tableChunks) {
|
|
1914
|
+
objects.push({
|
|
1915
|
+
path: normalizeResultPath(chunk.filePath),
|
|
1916
|
+
body: Buffer.from(chunk.html, "utf8"),
|
|
1917
|
+
contentType: htmlContentType,
|
|
1918
|
+
metadata: {
|
|
1919
|
+
chunkId: chunk.chunkId,
|
|
1920
|
+
chunkType: chunk.type,
|
|
1921
|
+
sourcePath: chunk.filePath
|
|
1922
|
+
}
|
|
1923
|
+
});
|
|
1924
|
+
}
|
|
1925
|
+
return objects;
|
|
1926
|
+
}
|
|
1927
|
+
function serializeChunks2(chunks) {
|
|
1928
|
+
return {
|
|
1929
|
+
chunks: chunks.map((chunk) => {
|
|
1930
|
+
const rawChunk = {
|
|
1931
|
+
chunkId: chunk.chunkId,
|
|
1932
|
+
type: chunk.type,
|
|
1933
|
+
contentSource: chunk.contentSource,
|
|
1934
|
+
content: chunk.content,
|
|
1935
|
+
path: chunk.path,
|
|
1936
|
+
metadata: chunk.metadata
|
|
1937
|
+
};
|
|
1938
|
+
if (chunk.type === "image" || chunk.type === "table") {
|
|
1939
|
+
return {
|
|
1940
|
+
...rawChunk,
|
|
1941
|
+
filePath: chunk.filePath,
|
|
1942
|
+
assetUrl: chunk.assetUrl
|
|
1943
|
+
};
|
|
1944
|
+
}
|
|
1945
|
+
return rawChunk;
|
|
1946
|
+
})
|
|
1947
|
+
};
|
|
1948
|
+
}
|
|
1949
|
+
async function writeParsedDocumentJsonObject(storage, params) {
|
|
1950
|
+
await storage.writeObject({
|
|
1951
|
+
documentId: params.documentId,
|
|
1952
|
+
revisionKey: params.revisionKey,
|
|
1953
|
+
path: params.path,
|
|
1954
|
+
body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
|
|
1955
|
+
contentType: jsonContentType
|
|
1956
|
+
});
|
|
1957
|
+
}
|
|
1958
|
+
function createJsonObject(path4, value) {
|
|
1959
|
+
return {
|
|
1960
|
+
path: path4,
|
|
1961
|
+
body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
|
|
1962
|
+
contentType: jsonContentType
|
|
1963
|
+
};
|
|
1964
|
+
}
|
|
1965
|
+
function createTextObject(path4, value, contentType) {
|
|
1966
|
+
return {
|
|
1967
|
+
path: path4,
|
|
1968
|
+
body: Buffer.from(value, "utf8"),
|
|
1969
|
+
contentType
|
|
1970
|
+
};
|
|
1971
|
+
}
|
|
1972
|
+
function inferContentTypeFromPath(filePath) {
|
|
1973
|
+
const lower = filePath.toLowerCase();
|
|
1974
|
+
if (lower.endsWith(".json")) return jsonContentType;
|
|
1975
|
+
if (lower.endsWith(".md")) return markdownContentType;
|
|
1976
|
+
if (lower.endsWith(".csv")) return csvContentType;
|
|
1977
|
+
if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
|
|
1978
|
+
if (lower.endsWith(".png")) return "image/png";
|
|
1979
|
+
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
|
1980
|
+
if (lower.endsWith(".gif")) return "image/gif";
|
|
1981
|
+
if (lower.endsWith(".webp")) return "image/webp";
|
|
1982
|
+
if (lower.endsWith(".svg")) return "image/svg+xml";
|
|
1983
|
+
return binaryContentType;
|
|
1984
|
+
}
|
|
1985
|
+
function normalizeResultPath(value) {
|
|
1986
|
+
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
1987
|
+
if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
|
|
1988
|
+
throw new Error(`Invalid parsed result path: ${value}`);
|
|
1989
|
+
}
|
|
1990
|
+
return normalized;
|
|
1991
|
+
}
|
|
1992
|
+
function isSdkMetadataPath(path4) {
|
|
1993
|
+
return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1509
1996
|
// src/knowledge/local-store.ts
|
|
1510
1997
|
import { createHash } from "crypto";
|
|
1511
1998
|
import os from "os";
|
|
@@ -1731,27 +2218,61 @@ function isMissingFileError2(error) {
|
|
|
1731
2218
|
// src/knowledge/knowledge.ts
|
|
1732
2219
|
var DEFAULT_READ_LIMIT = 12;
|
|
1733
2220
|
var MAX_READ_LIMIT = 40;
|
|
2221
|
+
var DEFAULT_PAGE_SIZE = 20;
|
|
2222
|
+
var MAX_PAGE_SIZE = 200;
|
|
1734
2223
|
var DEFAULT_GREP_LIMIT = 20;
|
|
1735
2224
|
var MAX_GREP_LIMIT = 50;
|
|
1736
2225
|
var DEFAULT_CONTEXT_CHARS = 80;
|
|
2226
|
+
var DEFAULT_REMOTE_SCAN_PAGE_SIZE = 100;
|
|
2227
|
+
var DEFAULT_MAX_SYNC_PAGES = 10;
|
|
2228
|
+
var DEFAULT_SYNC_DEADLINE_MS = 8e3;
|
|
2229
|
+
var DEFAULT_GREP_MAX_PAGES = 50;
|
|
2230
|
+
var DEFAULT_GREP_DEADLINE_MS = 8e3;
|
|
2231
|
+
var DEFAULT_OUTLINE_MAX_PAGES = 50;
|
|
2232
|
+
var DEFAULT_OUTLINE_DEADLINE_MS = 8e3;
|
|
1737
2233
|
var SAFE_LOCAL_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
1738
2234
|
var REMOTE_DOCUMENT_ID_PATTERN = /^doc[_-]/;
|
|
2235
|
+
var remoteReconstructionPageDirectory = ".knowhere-sdk/remote-reconstruction/pages";
|
|
1739
2236
|
var Knowledge = class _Knowledge {
|
|
1740
2237
|
client;
|
|
1741
2238
|
store;
|
|
2239
|
+
cacheDirectory;
|
|
2240
|
+
parsedStorageConfig;
|
|
1742
2241
|
constructor(client, options) {
|
|
1743
2242
|
this.client = client;
|
|
2243
|
+
this.cacheDirectory = options?.cacheDirectory;
|
|
2244
|
+
this.parsedStorageConfig = options?.parsedStorage;
|
|
1744
2245
|
this.store = new LocalKnowledgeStore(options?.cacheDirectory);
|
|
1745
2246
|
}
|
|
1746
2247
|
withCacheDirectory(cacheDirectory) {
|
|
1747
|
-
return new _Knowledge(this.client, {
|
|
2248
|
+
return new _Knowledge(this.client, {
|
|
2249
|
+
cacheDirectory,
|
|
2250
|
+
parsedStorage: this.parsedStorageConfig
|
|
2251
|
+
});
|
|
1748
2252
|
}
|
|
1749
|
-
|
|
1750
|
-
|
|
2253
|
+
withParsedStorage(parsedStorage) {
|
|
2254
|
+
return new _Knowledge(this.client, {
|
|
2255
|
+
cacheDirectory: this.cacheDirectory,
|
|
2256
|
+
parsedStorage
|
|
2257
|
+
});
|
|
2258
|
+
}
|
|
2259
|
+
async parseToLocalCache(params) {
|
|
2260
|
+
const loadedResult = params.storageAdapter ? await this.parseWithoutClientStorageAdapter(params) : await this.client.parse(params);
|
|
2261
|
+
const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter);
|
|
2262
|
+
const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result);
|
|
2263
|
+
const result = syncedStorage.result;
|
|
1751
2264
|
const document = await this.store.saveResult(result, {
|
|
1752
2265
|
localDocumentId: params.localDocumentId
|
|
1753
2266
|
});
|
|
1754
|
-
return {
|
|
2267
|
+
return {
|
|
2268
|
+
document,
|
|
2269
|
+
result,
|
|
2270
|
+
assetUrlsByFilePath: storedAssets.assetUrlsByFilePath
|
|
2271
|
+
};
|
|
2272
|
+
}
|
|
2273
|
+
/** @deprecated Use parseToLocalCache for local cache imports or the top-level client.parse for server-safe parsing. */
|
|
2274
|
+
async parse(params) {
|
|
2275
|
+
return this.parseToLocalCache(params);
|
|
1755
2276
|
}
|
|
1756
2277
|
async startParse(params) {
|
|
1757
2278
|
const job = await this.client.startParse(params);
|
|
@@ -1782,14 +2303,95 @@ var Knowledge = class _Knowledge {
|
|
|
1782
2303
|
results
|
|
1783
2304
|
};
|
|
1784
2305
|
}
|
|
1785
|
-
async
|
|
1786
|
-
const
|
|
2306
|
+
async loadJobResult(params) {
|
|
2307
|
+
const loadedResult = await this.client.jobs.load(params.jobId, {
|
|
1787
2308
|
verifyChecksum: params.verifyChecksum
|
|
1788
2309
|
});
|
|
2310
|
+
const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter);
|
|
2311
|
+
const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result);
|
|
2312
|
+
const result = syncedStorage.result;
|
|
2313
|
+
return {
|
|
2314
|
+
result,
|
|
2315
|
+
assetUrlsByFilePath: storedAssets.assetUrlsByFilePath
|
|
2316
|
+
};
|
|
2317
|
+
}
|
|
2318
|
+
async importJobResult(params) {
|
|
2319
|
+
const stored = await this.loadJobResult(params);
|
|
2320
|
+
const result = stored.result;
|
|
1789
2321
|
const document = await this.store.saveResult(result, {
|
|
1790
2322
|
localDocumentId: params.localDocumentId
|
|
1791
2323
|
});
|
|
1792
|
-
return {
|
|
2324
|
+
return {
|
|
2325
|
+
document,
|
|
2326
|
+
result,
|
|
2327
|
+
assetUrlsByFilePath: stored.assetUrlsByFilePath
|
|
2328
|
+
};
|
|
2329
|
+
}
|
|
2330
|
+
/** @deprecated Use importJobResult for local cache imports or loadJobResult for server-safe loads. */
|
|
2331
|
+
async cacheJobResult(params) {
|
|
2332
|
+
return this.importJobResult(params);
|
|
2333
|
+
}
|
|
2334
|
+
async syncParsedDocument(params) {
|
|
2335
|
+
const normalized = normalizeDocumentReference(params);
|
|
2336
|
+
const parsedStorageConfig = this.requireParsedStorageConfig();
|
|
2337
|
+
if (normalized.documentId) {
|
|
2338
|
+
return this.syncRemoteDocumentToParsedStorage({
|
|
2339
|
+
documentId: normalized.documentId,
|
|
2340
|
+
revisionKey: params.revisionKey,
|
|
2341
|
+
parsedStorageConfig
|
|
2342
|
+
});
|
|
2343
|
+
}
|
|
2344
|
+
if (normalized.jobId) {
|
|
2345
|
+
const loadedResult = await this.client.jobs.load(normalized.jobId);
|
|
2346
|
+
const synced2 = await syncParseResultToParsedDocumentStorage({
|
|
2347
|
+
result: loadedResult,
|
|
2348
|
+
storage: parsedStorageConfig.storage,
|
|
2349
|
+
documentId: loadedResult.documentId ?? normalized.localDocumentId ?? normalized.jobId,
|
|
2350
|
+
revisionKey: params.revisionKey ?? normalized.jobId
|
|
2351
|
+
});
|
|
2352
|
+
return {
|
|
2353
|
+
documentId: synced2.documentId,
|
|
2354
|
+
revisionKey: synced2.revisionKey,
|
|
2355
|
+
completed: true,
|
|
2356
|
+
commit: synced2.commit
|
|
2357
|
+
};
|
|
2358
|
+
}
|
|
2359
|
+
if (!normalized.localDocumentId) {
|
|
2360
|
+
throw new ValidationError("localDocumentId, documentId, or jobId is required");
|
|
2361
|
+
}
|
|
2362
|
+
const { result } = await this.store.loadResult(normalized.localDocumentId);
|
|
2363
|
+
const synced = await syncParseResultToParsedDocumentStorage({
|
|
2364
|
+
result,
|
|
2365
|
+
storage: parsedStorageConfig.storage,
|
|
2366
|
+
documentId: result.documentId ?? normalized.localDocumentId,
|
|
2367
|
+
revisionKey: params.revisionKey ?? result.jobId
|
|
2368
|
+
});
|
|
2369
|
+
return {
|
|
2370
|
+
documentId: synced.documentId,
|
|
2371
|
+
revisionKey: synced.revisionKey,
|
|
2372
|
+
completed: true,
|
|
2373
|
+
commit: synced.commit
|
|
2374
|
+
};
|
|
2375
|
+
}
|
|
2376
|
+
async syncParseResultToConfiguredStorage(result) {
|
|
2377
|
+
const parsedStorageConfig = this.parsedStorageConfig;
|
|
2378
|
+
if (!parsedStorageConfig) {
|
|
2379
|
+
return { result };
|
|
2380
|
+
}
|
|
2381
|
+
const synced = await syncParseResultToParsedDocumentStorage({
|
|
2382
|
+
result,
|
|
2383
|
+
storage: parsedStorageConfig.storage
|
|
2384
|
+
});
|
|
2385
|
+
return {
|
|
2386
|
+
result: synced.result,
|
|
2387
|
+
commit: synced.commit
|
|
2388
|
+
};
|
|
2389
|
+
}
|
|
2390
|
+
async parseWithoutClientStorageAdapter(params) {
|
|
2391
|
+
const parseParams = { ...params };
|
|
2392
|
+
delete parseParams.localDocumentId;
|
|
2393
|
+
delete parseParams.storageAdapter;
|
|
2394
|
+
return this.client.parse(parseParams);
|
|
1793
2395
|
}
|
|
1794
2396
|
async cacheDocument(params) {
|
|
1795
2397
|
if (!params.documentId) {
|
|
@@ -1799,7 +2401,7 @@ var Knowledge = class _Knowledge {
|
|
|
1799
2401
|
params.documentId
|
|
1800
2402
|
);
|
|
1801
2403
|
const jobId = await this.getPublishedDocumentJobId(params.documentId);
|
|
1802
|
-
return this.
|
|
2404
|
+
return this.importJobResult({
|
|
1803
2405
|
jobId,
|
|
1804
2406
|
localDocumentId: params.localDocumentId ?? existing?.localDocumentId ?? createLocalDocumentIdForRemote(params.documentId)
|
|
1805
2407
|
});
|
|
@@ -1836,7 +2438,7 @@ var Knowledge = class _Knowledge {
|
|
|
1836
2438
|
};
|
|
1837
2439
|
}
|
|
1838
2440
|
try {
|
|
1839
|
-
const cached = await this.
|
|
2441
|
+
const cached = await this.importJobResult({
|
|
1840
2442
|
jobId,
|
|
1841
2443
|
localDocumentId: trackedJob.localDocumentId
|
|
1842
2444
|
});
|
|
@@ -1861,6 +2463,13 @@ var Knowledge = class _Knowledge {
|
|
|
1861
2463
|
return this.store.listDocuments();
|
|
1862
2464
|
}
|
|
1863
2465
|
async getDocumentOutline(reference) {
|
|
2466
|
+
const normalized = normalizeDocumentReference(reference);
|
|
2467
|
+
if (normalized.documentId) {
|
|
2468
|
+
return this.getRemoteDocumentOutline(normalized.documentId, normalized.revisionKey);
|
|
2469
|
+
}
|
|
2470
|
+
if (normalized.localDocumentId && looksLikeRemoteDocumentId(normalized.localDocumentId) && !await this.store.getDocument(normalized.localDocumentId)) {
|
|
2471
|
+
return this.getRemoteDocumentOutline(normalized.localDocumentId, normalized.revisionKey);
|
|
2472
|
+
}
|
|
1864
2473
|
const { document, result } = await this.loadReadableResult(reference);
|
|
1865
2474
|
const chunks = indexChunks(result);
|
|
1866
2475
|
const sections = buildFlatSections(result, chunks);
|
|
@@ -1876,6 +2485,14 @@ var Knowledge = class _Knowledge {
|
|
|
1876
2485
|
};
|
|
1877
2486
|
}
|
|
1878
2487
|
async readChunks(params) {
|
|
2488
|
+
validateReadParams(params);
|
|
2489
|
+
const normalized = normalizeDocumentReference(params);
|
|
2490
|
+
if (normalized.documentId) {
|
|
2491
|
+
return this.readRemoteDocumentChunks(normalized.documentId, params);
|
|
2492
|
+
}
|
|
2493
|
+
if (normalized.localDocumentId && looksLikeRemoteDocumentId(normalized.localDocumentId) && !await this.store.getDocument(normalized.localDocumentId)) {
|
|
2494
|
+
return this.readRemoteDocumentChunks(normalized.localDocumentId, params);
|
|
2495
|
+
}
|
|
1879
2496
|
const { document, result } = await this.loadReadableResult(params);
|
|
1880
2497
|
const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT);
|
|
1881
2498
|
const chunks = indexChunks(result).filter((chunk) => matchesReadScope(chunk, params));
|
|
@@ -1892,6 +2509,13 @@ var Knowledge = class _Knowledge {
|
|
|
1892
2509
|
if (!params.pattern) {
|
|
1893
2510
|
throw new ValidationError("pattern is required");
|
|
1894
2511
|
}
|
|
2512
|
+
const normalized = normalizeDocumentReference(params);
|
|
2513
|
+
if (normalized.documentId) {
|
|
2514
|
+
return this.grepRemoteDocumentChunks(normalized.documentId, params);
|
|
2515
|
+
}
|
|
2516
|
+
if (normalized.localDocumentId && looksLikeRemoteDocumentId(normalized.localDocumentId) && !await this.store.getDocument(normalized.localDocumentId)) {
|
|
2517
|
+
return this.grepRemoteDocumentChunks(normalized.localDocumentId, params);
|
|
2518
|
+
}
|
|
1895
2519
|
const { document, result } = await this.loadReadableResult(params);
|
|
1896
2520
|
const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT);
|
|
1897
2521
|
const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS;
|
|
@@ -1971,7 +2595,7 @@ var Knowledge = class _Knowledge {
|
|
|
1971
2595
|
});
|
|
1972
2596
|
}
|
|
1973
2597
|
if (normalized.jobId) {
|
|
1974
|
-
return this.
|
|
2598
|
+
return this.importJobResult({
|
|
1975
2599
|
jobId: normalized.jobId,
|
|
1976
2600
|
localDocumentId: normalized.localDocumentId
|
|
1977
2601
|
});
|
|
@@ -1991,21 +2615,657 @@ var Knowledge = class _Knowledge {
|
|
|
1991
2615
|
}
|
|
1992
2616
|
throw new Error(`Local Knowhere document not found: ${normalized.localDocumentId}`);
|
|
1993
2617
|
}
|
|
1994
|
-
async
|
|
1995
|
-
const
|
|
1996
|
-
|
|
2618
|
+
async readRemoteDocumentChunks(documentId, params) {
|
|
2619
|
+
const explicitRevisionKey = params.revisionKey ?? params.jobId;
|
|
2620
|
+
if (explicitRevisionKey) {
|
|
2621
|
+
const stored2 = await this.tryReadChunksFromParsedStorage({
|
|
2622
|
+
documentId,
|
|
2623
|
+
revisionKey: explicitRevisionKey,
|
|
2624
|
+
params
|
|
2625
|
+
});
|
|
2626
|
+
if (stored2) {
|
|
2627
|
+
return stored2;
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
if (hasScanReadFilters(params)) {
|
|
2631
|
+
return this.scanRemoteDocumentChunks(documentId, params);
|
|
2632
|
+
}
|
|
2633
|
+
const pageParams = getReadPageParams(params);
|
|
2634
|
+
const remotePage = await this.client.documents.listChunks(documentId, {
|
|
2635
|
+
page: pageParams.page,
|
|
2636
|
+
pageSize: pageParams.pageSize,
|
|
2637
|
+
chunkType: params.chunkType,
|
|
2638
|
+
includeAssetUrls: true
|
|
2639
|
+
});
|
|
2640
|
+
const revisionKey = getRemoteRevisionKey(remotePage);
|
|
2641
|
+
const stored = await this.tryReadChunksFromParsedStorage({
|
|
2642
|
+
documentId,
|
|
2643
|
+
revisionKey,
|
|
2644
|
+
params
|
|
2645
|
+
});
|
|
2646
|
+
if (stored) {
|
|
2647
|
+
return stored;
|
|
2648
|
+
}
|
|
2649
|
+
const chunks = remotePage.chunks.map((chunk) => toRemoteReadChunk(chunk));
|
|
2650
|
+
this.scheduleParsedStorageSync(documentId, revisionKey);
|
|
2651
|
+
return {
|
|
2652
|
+
document: createRemoteKnowledgeDocument({
|
|
2653
|
+
documentId,
|
|
2654
|
+
response: remotePage,
|
|
2655
|
+
revisionKey
|
|
2656
|
+
}),
|
|
2657
|
+
chunks,
|
|
2658
|
+
page: remotePage.pagination.page,
|
|
2659
|
+
pageSize: remotePage.pagination.pageSize,
|
|
2660
|
+
totalChunks: remotePage.pagination.total,
|
|
2661
|
+
totalPages: remotePage.pagination.totalPages
|
|
2662
|
+
};
|
|
1997
2663
|
}
|
|
1998
|
-
async
|
|
1999
|
-
const
|
|
2664
|
+
async scanRemoteDocumentChunks(documentId, params) {
|
|
2665
|
+
const explicitRevisionKey = params.revisionKey ?? params.jobId;
|
|
2666
|
+
if (explicitRevisionKey) {
|
|
2667
|
+
const stored2 = await this.tryReadChunksFromParsedStorage({
|
|
2668
|
+
documentId,
|
|
2669
|
+
revisionKey: explicitRevisionKey,
|
|
2670
|
+
params
|
|
2671
|
+
});
|
|
2672
|
+
if (stored2) {
|
|
2673
|
+
return stored2;
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT);
|
|
2677
|
+
const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
|
|
2678
|
+
const firstPage = await this.client.documents.listChunks(documentId, {
|
|
2000
2679
|
page: 1,
|
|
2001
|
-
pageSize:
|
|
2680
|
+
pageSize: limits.remotePageSize,
|
|
2681
|
+
chunkType: params.chunkType,
|
|
2682
|
+
includeAssetUrls: true
|
|
2002
2683
|
});
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2684
|
+
const revisionKey = getRemoteRevisionKey(firstPage);
|
|
2685
|
+
const stored = await this.tryReadChunksFromParsedStorage({
|
|
2686
|
+
documentId,
|
|
2687
|
+
revisionKey,
|
|
2688
|
+
params
|
|
2689
|
+
});
|
|
2690
|
+
if (stored) {
|
|
2691
|
+
return stored;
|
|
2007
2692
|
}
|
|
2008
|
-
|
|
2693
|
+
const selected = [];
|
|
2694
|
+
let scannedPages = 0;
|
|
2695
|
+
let page = 1;
|
|
2696
|
+
let response = firstPage;
|
|
2697
|
+
const startedAt = Date.now();
|
|
2698
|
+
while (true) {
|
|
2699
|
+
scannedPages += 1;
|
|
2700
|
+
selected.push(...response.chunks.filter((chunk) => matchesRemoteReadScope(chunk, params)));
|
|
2701
|
+
if (selected.length >= limit || response.pagination.page >= response.pagination.totalPages || scannedPages >= limits.outlineMaxPages || Date.now() - startedAt >= limits.outlineDeadlineMs) {
|
|
2702
|
+
break;
|
|
2703
|
+
}
|
|
2704
|
+
page += 1;
|
|
2705
|
+
response = await this.client.documents.listChunks(documentId, {
|
|
2706
|
+
page,
|
|
2707
|
+
pageSize: limits.remotePageSize,
|
|
2708
|
+
chunkType: params.chunkType,
|
|
2709
|
+
includeAssetUrls: true
|
|
2710
|
+
});
|
|
2711
|
+
}
|
|
2712
|
+
const chunks = selected.slice(0, limit).map((chunk) => toRemoteReadChunk(chunk));
|
|
2713
|
+
this.scheduleParsedStorageSync(documentId, revisionKey);
|
|
2714
|
+
return {
|
|
2715
|
+
document: createRemoteKnowledgeDocument({
|
|
2716
|
+
documentId,
|
|
2717
|
+
response: firstPage,
|
|
2718
|
+
revisionKey
|
|
2719
|
+
}),
|
|
2720
|
+
chunks,
|
|
2721
|
+
nextChunk: selected.length > limit ? toChunkPosition(selected[limit]) : void 0,
|
|
2722
|
+
totalChunks: firstPage.pagination.total,
|
|
2723
|
+
totalPages: firstPage.pagination.totalPages
|
|
2724
|
+
};
|
|
2725
|
+
}
|
|
2726
|
+
async grepRemoteDocumentChunks(documentId, params) {
|
|
2727
|
+
const explicitRevisionKey = params.revisionKey ?? params.jobId;
|
|
2728
|
+
if (explicitRevisionKey && !params.continuationCursor) {
|
|
2729
|
+
const stored2 = await this.tryGrepChunksFromParsedStorage({
|
|
2730
|
+
documentId,
|
|
2731
|
+
revisionKey: explicitRevisionKey,
|
|
2732
|
+
params
|
|
2733
|
+
});
|
|
2734
|
+
if (stored2) {
|
|
2735
|
+
return stored2;
|
|
2736
|
+
}
|
|
2737
|
+
}
|
|
2738
|
+
const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT);
|
|
2739
|
+
const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS;
|
|
2740
|
+
const matcher = createMatcher(params);
|
|
2741
|
+
const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
|
|
2742
|
+
const cursor = parseContinuationCursor(params.continuationCursor);
|
|
2743
|
+
const startPage = cursor?.documentId === documentId ? cursor.nextPage : 1;
|
|
2744
|
+
const firstPage = await this.client.documents.listChunks(documentId, {
|
|
2745
|
+
page: startPage,
|
|
2746
|
+
pageSize: limits.remotePageSize,
|
|
2747
|
+
chunkType: params.chunkType,
|
|
2748
|
+
includeAssetUrls: true
|
|
2749
|
+
});
|
|
2750
|
+
const revisionKey = getRemoteRevisionKey(firstPage);
|
|
2751
|
+
const stored = startPage === 1 ? await this.tryGrepChunksFromParsedStorage({
|
|
2752
|
+
documentId,
|
|
2753
|
+
revisionKey,
|
|
2754
|
+
params
|
|
2755
|
+
}) : null;
|
|
2756
|
+
if (stored) {
|
|
2757
|
+
return stored;
|
|
2758
|
+
}
|
|
2759
|
+
const matches = [];
|
|
2760
|
+
let scannedChunks = 0;
|
|
2761
|
+
let scannedPages = 0;
|
|
2762
|
+
let response = firstPage;
|
|
2763
|
+
let page = startPage;
|
|
2764
|
+
const startedAt = Date.now();
|
|
2765
|
+
while (true) {
|
|
2766
|
+
scannedPages += 1;
|
|
2767
|
+
for (const [chunkIndex, chunk] of response.chunks.entries()) {
|
|
2768
|
+
if (response.pagination.page === startPage && cursor?.nextChunkIndex !== void 0 && chunkIndex < cursor.nextChunkIndex) {
|
|
2769
|
+
continue;
|
|
2770
|
+
}
|
|
2771
|
+
if (!matchesGrepScope(toIndexedRemoteChunk(chunk), params)) {
|
|
2772
|
+
continue;
|
|
2773
|
+
}
|
|
2774
|
+
scannedChunks += 1;
|
|
2775
|
+
const chunkMatches = matcher(chunk.content ?? "");
|
|
2776
|
+
for (const [matchIndex, match] of chunkMatches.entries()) {
|
|
2777
|
+
if (response.pagination.page === startPage && cursor?.nextChunkIndex === chunkIndex && cursor.nextMatchIndex !== void 0 && matchIndex < cursor.nextMatchIndex) {
|
|
2778
|
+
continue;
|
|
2779
|
+
}
|
|
2780
|
+
matches.push({
|
|
2781
|
+
position: toChunkPosition(chunk),
|
|
2782
|
+
chunkId: chunk.chunkId,
|
|
2783
|
+
chunkType: chunk.chunkType,
|
|
2784
|
+
sectionPath: chunk.sectionPath ?? "",
|
|
2785
|
+
sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? "",
|
|
2786
|
+
filePath: chunk.filePath ?? void 0,
|
|
2787
|
+
startOffset: match.startOffset,
|
|
2788
|
+
endOffset: match.endOffset,
|
|
2789
|
+
snippet: buildSnippet(
|
|
2790
|
+
chunk.content ?? "",
|
|
2791
|
+
match.startOffset,
|
|
2792
|
+
match.endOffset,
|
|
2793
|
+
contextChars
|
|
2794
|
+
)
|
|
2795
|
+
});
|
|
2796
|
+
if (matches.length >= maxResults) {
|
|
2797
|
+
const continuationCursor = createGrepContinuationCursor({
|
|
2798
|
+
documentId,
|
|
2799
|
+
revisionKey,
|
|
2800
|
+
response,
|
|
2801
|
+
chunkIndex,
|
|
2802
|
+
matchIndex,
|
|
2803
|
+
chunkMatchCount: chunkMatches.length
|
|
2804
|
+
});
|
|
2805
|
+
this.scheduleParsedStorageSync(documentId, revisionKey);
|
|
2806
|
+
return {
|
|
2807
|
+
document: createRemoteKnowledgeDocument({
|
|
2808
|
+
documentId,
|
|
2809
|
+
response: firstPage,
|
|
2810
|
+
revisionKey
|
|
2811
|
+
}),
|
|
2812
|
+
matches,
|
|
2813
|
+
scannedChunks,
|
|
2814
|
+
truncated: continuationCursor !== void 0,
|
|
2815
|
+
continuationCursor
|
|
2816
|
+
};
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
}
|
|
2820
|
+
const shouldStop = response.pagination.page >= response.pagination.totalPages || scannedPages >= limits.grepMaxPages || Date.now() - startedAt >= limits.grepDeadlineMs;
|
|
2821
|
+
if (shouldStop) {
|
|
2822
|
+
const truncated = response.pagination.page < response.pagination.totalPages;
|
|
2823
|
+
this.scheduleParsedStorageSync(documentId, revisionKey);
|
|
2824
|
+
return {
|
|
2825
|
+
document: createRemoteKnowledgeDocument({
|
|
2826
|
+
documentId,
|
|
2827
|
+
response: firstPage,
|
|
2828
|
+
revisionKey
|
|
2829
|
+
}),
|
|
2830
|
+
matches,
|
|
2831
|
+
scannedChunks,
|
|
2832
|
+
truncated,
|
|
2833
|
+
continuationCursor: truncated ? createContinuationCursor({
|
|
2834
|
+
documentId,
|
|
2835
|
+
revisionKey,
|
|
2836
|
+
nextPage: response.pagination.page + 1
|
|
2837
|
+
}) : void 0
|
|
2838
|
+
};
|
|
2839
|
+
}
|
|
2840
|
+
page += 1;
|
|
2841
|
+
response = await this.client.documents.listChunks(documentId, {
|
|
2842
|
+
page,
|
|
2843
|
+
pageSize: limits.remotePageSize,
|
|
2844
|
+
chunkType: params.chunkType,
|
|
2845
|
+
includeAssetUrls: true
|
|
2846
|
+
});
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
async getRemoteDocumentOutline(documentId, revisionKeyOverride) {
|
|
2850
|
+
if (revisionKeyOverride) {
|
|
2851
|
+
const stored2 = await this.tryReadOutlineFromParsedStorage(documentId, revisionKeyOverride);
|
|
2852
|
+
if (stored2) {
|
|
2853
|
+
return stored2;
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
|
|
2857
|
+
const firstPage = await this.client.documents.listChunks(documentId, {
|
|
2858
|
+
page: 1,
|
|
2859
|
+
pageSize: limits.remotePageSize,
|
|
2860
|
+
includeAssetUrls: true
|
|
2861
|
+
});
|
|
2862
|
+
const revisionKey = getRemoteRevisionKey(firstPage);
|
|
2863
|
+
const stored = await this.tryReadOutlineFromParsedStorage(documentId, revisionKey);
|
|
2864
|
+
if (stored) {
|
|
2865
|
+
return stored;
|
|
2866
|
+
}
|
|
2867
|
+
const chunks = [];
|
|
2868
|
+
let page = 1;
|
|
2869
|
+
let response = firstPage;
|
|
2870
|
+
let scannedPages = 0;
|
|
2871
|
+
const startedAt = Date.now();
|
|
2872
|
+
while (true) {
|
|
2873
|
+
scannedPages += 1;
|
|
2874
|
+
chunks.push(...response.chunks.map(toIndexedRemoteChunk));
|
|
2875
|
+
if (response.pagination.page >= response.pagination.totalPages || scannedPages >= limits.outlineMaxPages || Date.now() - startedAt >= limits.outlineDeadlineMs) {
|
|
2876
|
+
break;
|
|
2877
|
+
}
|
|
2878
|
+
page += 1;
|
|
2879
|
+
response = await this.client.documents.listChunks(documentId, {
|
|
2880
|
+
page,
|
|
2881
|
+
pageSize: limits.remotePageSize,
|
|
2882
|
+
includeAssetUrls: true
|
|
2883
|
+
});
|
|
2884
|
+
}
|
|
2885
|
+
const truncated = response.pagination.page < response.pagination.totalPages;
|
|
2886
|
+
const sections = buildFlatSectionsFromIndexedChunks(chunks);
|
|
2887
|
+
this.scheduleParsedStorageSync(documentId, revisionKey);
|
|
2888
|
+
return {
|
|
2889
|
+
document: createRemoteKnowledgeDocument({
|
|
2890
|
+
documentId,
|
|
2891
|
+
response: firstPage,
|
|
2892
|
+
revisionKey,
|
|
2893
|
+
typeCounts: countIndexedTypes(chunks)
|
|
2894
|
+
}),
|
|
2895
|
+
totalChunks: firstPage.pagination.total,
|
|
2896
|
+
typeCounts: countIndexedTypes(chunks),
|
|
2897
|
+
sections,
|
|
2898
|
+
sectionTree: nestSections(sections),
|
|
2899
|
+
truncated,
|
|
2900
|
+
continuationCursor: truncated ? createContinuationCursor({
|
|
2901
|
+
documentId,
|
|
2902
|
+
revisionKey,
|
|
2903
|
+
nextPage: response.pagination.page + 1
|
|
2904
|
+
}) : void 0
|
|
2905
|
+
};
|
|
2906
|
+
}
|
|
2907
|
+
async tryReadChunksFromParsedStorage(params) {
|
|
2908
|
+
const storedResult = await this.readCommittedParsedStorageResult(
|
|
2909
|
+
params.documentId,
|
|
2910
|
+
params.revisionKey
|
|
2911
|
+
);
|
|
2912
|
+
if (!storedResult) return null;
|
|
2913
|
+
const pageParams = getReadPageParams(params.params);
|
|
2914
|
+
const scopedChunks = storedResult.chunks.filter((chunk) => !params.params.chunkType || chunk.chunkType === params.params.chunkType).filter((chunk) => matchesReadScope(chunk, params.params));
|
|
2915
|
+
const selected = hasScanReadFilters(params.params) ? selectReadWindow(
|
|
2916
|
+
scopedChunks,
|
|
2917
|
+
params.params,
|
|
2918
|
+
clampLimit(params.params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT)
|
|
2919
|
+
) : scopedChunks.slice(
|
|
2920
|
+
(pageParams.page - 1) * pageParams.pageSize,
|
|
2921
|
+
pageParams.page * pageParams.pageSize
|
|
2922
|
+
);
|
|
2923
|
+
const visibleChunks = await applyStoredAssetUrlsToReadChunks({
|
|
2924
|
+
storageConfig: this.requireParsedStorageConfig(),
|
|
2925
|
+
documentId: params.documentId,
|
|
2926
|
+
revisionKey: params.revisionKey,
|
|
2927
|
+
chunks: selected.map(toReadChunk)
|
|
2928
|
+
});
|
|
2929
|
+
const totalChunks = scopedChunks.length;
|
|
2930
|
+
return {
|
|
2931
|
+
document: createStoredKnowledgeDocument({
|
|
2932
|
+
documentId: params.documentId,
|
|
2933
|
+
storedResult,
|
|
2934
|
+
revisionKey: params.revisionKey
|
|
2935
|
+
}),
|
|
2936
|
+
chunks: visibleChunks,
|
|
2937
|
+
page: hasScanReadFilters(params.params) ? void 0 : pageParams.page,
|
|
2938
|
+
pageSize: hasScanReadFilters(params.params) ? void 0 : pageParams.pageSize,
|
|
2939
|
+
totalChunks,
|
|
2940
|
+
totalPages: Math.max(1, Math.ceil(totalChunks / pageParams.pageSize)),
|
|
2941
|
+
nextChunk: getNextChunkPosition(scopedChunks, selected)
|
|
2942
|
+
};
|
|
2943
|
+
}
|
|
2944
|
+
async tryGrepChunksFromParsedStorage(params) {
|
|
2945
|
+
const storedResult = await this.readCommittedParsedStorageResult(
|
|
2946
|
+
params.documentId,
|
|
2947
|
+
params.revisionKey
|
|
2948
|
+
);
|
|
2949
|
+
if (!storedResult) return null;
|
|
2950
|
+
const maxResults = clampLimit(params.params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT);
|
|
2951
|
+
const contextChars = params.params.contextChars ?? DEFAULT_CONTEXT_CHARS;
|
|
2952
|
+
const matcher = createMatcher(params.params);
|
|
2953
|
+
const scopedChunks = storedResult.chunks.filter((chunk) => !params.params.chunkType || chunk.chunkType === params.params.chunkType).filter((chunk) => matchesGrepScope(chunk, params.params));
|
|
2954
|
+
const matches = [];
|
|
2955
|
+
let scannedChunks = 0;
|
|
2956
|
+
for (const chunk of scopedChunks) {
|
|
2957
|
+
scannedChunks += 1;
|
|
2958
|
+
for (const match of matcher(chunk.content)) {
|
|
2959
|
+
matches.push({
|
|
2960
|
+
position: chunk.position,
|
|
2961
|
+
chunkId: chunk.chunkId,
|
|
2962
|
+
chunkType: chunk.chunkType,
|
|
2963
|
+
sectionPath: chunk.sectionPath,
|
|
2964
|
+
sourceChunkPath: chunk.sourceChunkPath,
|
|
2965
|
+
filePath: chunk.filePath,
|
|
2966
|
+
startOffset: match.startOffset,
|
|
2967
|
+
endOffset: match.endOffset,
|
|
2968
|
+
snippet: buildSnippet(chunk.content, match.startOffset, match.endOffset, contextChars)
|
|
2969
|
+
});
|
|
2970
|
+
if (matches.length >= maxResults) {
|
|
2971
|
+
return {
|
|
2972
|
+
document: createStoredKnowledgeDocument({
|
|
2973
|
+
documentId: params.documentId,
|
|
2974
|
+
storedResult,
|
|
2975
|
+
revisionKey: params.revisionKey
|
|
2976
|
+
}),
|
|
2977
|
+
matches,
|
|
2978
|
+
scannedChunks,
|
|
2979
|
+
truncated: true
|
|
2980
|
+
};
|
|
2981
|
+
}
|
|
2982
|
+
}
|
|
2983
|
+
}
|
|
2984
|
+
return {
|
|
2985
|
+
document: createStoredKnowledgeDocument({
|
|
2986
|
+
documentId: params.documentId,
|
|
2987
|
+
storedResult,
|
|
2988
|
+
revisionKey: params.revisionKey
|
|
2989
|
+
}),
|
|
2990
|
+
matches,
|
|
2991
|
+
scannedChunks,
|
|
2992
|
+
truncated: false
|
|
2993
|
+
};
|
|
2994
|
+
}
|
|
2995
|
+
async tryReadOutlineFromParsedStorage(documentId, revisionKey) {
|
|
2996
|
+
const storedResult = await this.readCommittedParsedStorageResult(documentId, revisionKey);
|
|
2997
|
+
if (!storedResult) return null;
|
|
2998
|
+
const sections = storedResult.docNav?.sections && storedResult.docNav.sections.length > 0 ? flattenSections(
|
|
2999
|
+
storedResult.docNav.sections.map(
|
|
3000
|
+
(section) => toKnowledgeSection(
|
|
3001
|
+
section,
|
|
3002
|
+
storedResult.chunks,
|
|
3003
|
+
storedResult.manifest.sourceFileName
|
|
3004
|
+
)
|
|
3005
|
+
)
|
|
3006
|
+
) : buildFlatSectionsFromIndexedChunks(storedResult.chunks);
|
|
3007
|
+
return {
|
|
3008
|
+
document: createStoredKnowledgeDocument({
|
|
3009
|
+
documentId,
|
|
3010
|
+
storedResult,
|
|
3011
|
+
revisionKey
|
|
3012
|
+
}),
|
|
3013
|
+
totalChunks: storedResult.chunks.length,
|
|
3014
|
+
typeCounts: storedResult.typeCounts,
|
|
3015
|
+
sections,
|
|
3016
|
+
sectionTree: storedResult.docNav?.sections && storedResult.docNav.sections.length > 0 ? storedResult.docNav.sections.map(
|
|
3017
|
+
(section) => toKnowledgeSection(
|
|
3018
|
+
section,
|
|
3019
|
+
storedResult.chunks,
|
|
3020
|
+
storedResult.manifest.sourceFileName
|
|
3021
|
+
)
|
|
3022
|
+
) : nestSections(sections),
|
|
3023
|
+
truncated: false
|
|
3024
|
+
};
|
|
3025
|
+
}
|
|
3026
|
+
async readCommittedParsedStorageResult(documentId, revisionKey) {
|
|
3027
|
+
const parsedStorageConfig = this.parsedStorageConfig;
|
|
3028
|
+
if (!parsedStorageConfig) {
|
|
3029
|
+
return null;
|
|
3030
|
+
}
|
|
3031
|
+
const commit = await readParsedDocumentCommit({
|
|
3032
|
+
storage: parsedStorageConfig.storage,
|
|
3033
|
+
documentId,
|
|
3034
|
+
revisionKey
|
|
3035
|
+
});
|
|
3036
|
+
if (!commit) {
|
|
3037
|
+
return null;
|
|
3038
|
+
}
|
|
3039
|
+
const manifestObject = await parsedStorageConfig.storage.readObject({
|
|
3040
|
+
documentId,
|
|
3041
|
+
revisionKey,
|
|
3042
|
+
path: "manifest.json"
|
|
3043
|
+
});
|
|
3044
|
+
const chunksObject = await parsedStorageConfig.storage.readObject({
|
|
3045
|
+
documentId,
|
|
3046
|
+
revisionKey,
|
|
3047
|
+
path: "chunks.json"
|
|
3048
|
+
});
|
|
3049
|
+
if (!manifestObject || !chunksObject) {
|
|
3050
|
+
return null;
|
|
3051
|
+
}
|
|
3052
|
+
const manifest = readJsonObject(manifestObject);
|
|
3053
|
+
const chunks = parseStoredResultChunks({
|
|
3054
|
+
payload: readJsonObject(chunksObject),
|
|
3055
|
+
sourceFileName: manifest.sourceFileName
|
|
3056
|
+
});
|
|
3057
|
+
const docNavObject = await parsedStorageConfig.storage.readObject({
|
|
3058
|
+
documentId,
|
|
3059
|
+
revisionKey,
|
|
3060
|
+
path: "doc_nav.json"
|
|
3061
|
+
});
|
|
3062
|
+
const docNav = docNavObject ? readJsonObject(docNavObject) : void 0;
|
|
3063
|
+
return {
|
|
3064
|
+
commit,
|
|
3065
|
+
manifest,
|
|
3066
|
+
chunks,
|
|
3067
|
+
docNav: docNav?.sections ? { sections: docNav.sections } : void 0,
|
|
3068
|
+
typeCounts: countIndexedTypes(chunks)
|
|
3069
|
+
};
|
|
3070
|
+
}
|
|
3071
|
+
scheduleParsedStorageSync(documentId, revisionKey) {
|
|
3072
|
+
const parsedStorageConfig = this.parsedStorageConfig;
|
|
3073
|
+
if (!parsedStorageConfig) {
|
|
3074
|
+
return;
|
|
3075
|
+
}
|
|
3076
|
+
const scheduler = parsedStorageConfig.scheduler ?? defaultParsedStorageScheduler;
|
|
3077
|
+
const task = async () => {
|
|
3078
|
+
try {
|
|
3079
|
+
await this.syncRemoteDocumentToParsedStorage({
|
|
3080
|
+
documentId,
|
|
3081
|
+
revisionKey,
|
|
3082
|
+
parsedStorageConfig
|
|
3083
|
+
});
|
|
3084
|
+
} catch (error) {
|
|
3085
|
+
await parsedStorageConfig.storage.writeSyncProgress({
|
|
3086
|
+
documentId,
|
|
3087
|
+
revisionKey,
|
|
3088
|
+
nextChunkPage: 1,
|
|
3089
|
+
status: "failed",
|
|
3090
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3091
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3092
|
+
});
|
|
3093
|
+
}
|
|
3094
|
+
};
|
|
3095
|
+
void Promise.resolve(scheduler.schedule(task)).catch(() => void 0);
|
|
3096
|
+
}
|
|
3097
|
+
async syncRemoteDocumentToParsedStorage(params) {
|
|
3098
|
+
const firstPage = await this.client.documents.listChunks(params.documentId, {
|
|
3099
|
+
page: 1,
|
|
3100
|
+
pageSize: 1,
|
|
3101
|
+
includeAssetUrls: true
|
|
3102
|
+
});
|
|
3103
|
+
const revisionKey = params.revisionKey ?? getRemoteRevisionKey(firstPage);
|
|
3104
|
+
if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey) {
|
|
3105
|
+
return this.syncRemoteDocumentToParsedStorage({
|
|
3106
|
+
documentId: params.documentId,
|
|
3107
|
+
parsedStorageConfig: params.parsedStorageConfig
|
|
3108
|
+
});
|
|
3109
|
+
}
|
|
3110
|
+
if (firstPage.jobId) {
|
|
3111
|
+
try {
|
|
3112
|
+
const loadedResult = await this.client.jobs.load(firstPage.jobId);
|
|
3113
|
+
const synced = await syncParseResultToParsedDocumentStorage({
|
|
3114
|
+
result: loadedResult,
|
|
3115
|
+
storage: params.parsedStorageConfig.storage,
|
|
3116
|
+
documentId: params.documentId,
|
|
3117
|
+
revisionKey,
|
|
3118
|
+
source: "resultZip"
|
|
3119
|
+
});
|
|
3120
|
+
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
3121
|
+
documentId: params.documentId,
|
|
3122
|
+
revisionKey,
|
|
3123
|
+
status: "completed",
|
|
3124
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3125
|
+
});
|
|
3126
|
+
return {
|
|
3127
|
+
documentId: synced.documentId,
|
|
3128
|
+
revisionKey: synced.revisionKey,
|
|
3129
|
+
completed: true,
|
|
3130
|
+
commit: synced.commit
|
|
3131
|
+
};
|
|
3132
|
+
} catch {
|
|
3133
|
+
}
|
|
3134
|
+
}
|
|
3135
|
+
return this.syncRemoteDocumentChunksToParsedStorage({
|
|
3136
|
+
documentId: params.documentId,
|
|
3137
|
+
revisionKey,
|
|
3138
|
+
firstPage,
|
|
3139
|
+
parsedStorageConfig: params.parsedStorageConfig
|
|
3140
|
+
});
|
|
3141
|
+
}
|
|
3142
|
+
async syncRemoteDocumentChunksToParsedStorage(params) {
|
|
3143
|
+
const limits = getParsedStorageLimits(params.parsedStorageConfig.limits);
|
|
3144
|
+
const existingProgress = await params.parsedStorageConfig.storage.readSyncProgress({
|
|
3145
|
+
documentId: params.documentId,
|
|
3146
|
+
revisionKey: params.revisionKey
|
|
3147
|
+
});
|
|
3148
|
+
let nextPage = existingProgress?.status === "running" ? existingProgress.nextChunkPage ?? 1 : 1;
|
|
3149
|
+
let response = nextPage === 1 && params.firstPage.pagination.pageSize === limits.remotePageSize ? params.firstPage : await this.client.documents.listChunks(params.documentId, {
|
|
3150
|
+
page: nextPage,
|
|
3151
|
+
pageSize: limits.remotePageSize,
|
|
3152
|
+
includeAssetUrls: true
|
|
3153
|
+
});
|
|
3154
|
+
const totalPages = response.pagination.totalPages;
|
|
3155
|
+
let syncedPages = 0;
|
|
3156
|
+
const startedAt = Date.now();
|
|
3157
|
+
while (nextPage <= totalPages) {
|
|
3158
|
+
if (syncedPages > 0) {
|
|
3159
|
+
response = await this.client.documents.listChunks(params.documentId, {
|
|
3160
|
+
page: nextPage,
|
|
3161
|
+
pageSize: limits.remotePageSize,
|
|
3162
|
+
includeAssetUrls: true
|
|
3163
|
+
});
|
|
3164
|
+
}
|
|
3165
|
+
await params.parsedStorageConfig.storage.writeObject({
|
|
3166
|
+
documentId: params.documentId,
|
|
3167
|
+
revisionKey: params.revisionKey,
|
|
3168
|
+
path: getRemoteReconstructionPagePath(response.pagination.page),
|
|
3169
|
+
body: Buffer.from(JSON.stringify({ chunks: response.chunks }), "utf8"),
|
|
3170
|
+
contentType: "application/json; charset=utf-8"
|
|
3171
|
+
});
|
|
3172
|
+
nextPage += 1;
|
|
3173
|
+
syncedPages += 1;
|
|
3174
|
+
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
3175
|
+
documentId: params.documentId,
|
|
3176
|
+
revisionKey: params.revisionKey,
|
|
3177
|
+
nextChunkPage: nextPage,
|
|
3178
|
+
status: "running",
|
|
3179
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3180
|
+
});
|
|
3181
|
+
if (syncedPages >= limits.maxPagesPerSync || Date.now() - startedAt >= limits.syncDeadlineMs) {
|
|
3182
|
+
return {
|
|
3183
|
+
documentId: params.documentId,
|
|
3184
|
+
revisionKey: params.revisionKey,
|
|
3185
|
+
completed: false
|
|
3186
|
+
};
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3189
|
+
const chunks = await this.readRemoteReconstructionChunks({
|
|
3190
|
+
documentId: params.documentId,
|
|
3191
|
+
revisionKey: params.revisionKey,
|
|
3192
|
+
totalPages,
|
|
3193
|
+
parsedStorageConfig: params.parsedStorageConfig
|
|
3194
|
+
});
|
|
3195
|
+
if (!chunks) {
|
|
3196
|
+
return {
|
|
3197
|
+
documentId: params.documentId,
|
|
3198
|
+
revisionKey: params.revisionKey,
|
|
3199
|
+
completed: false
|
|
3200
|
+
};
|
|
3201
|
+
}
|
|
3202
|
+
const synced = await syncParseResultToParsedDocumentStorage({
|
|
3203
|
+
result: createMinimalParseResultFromRemoteChunks({
|
|
3204
|
+
documentId: params.documentId,
|
|
3205
|
+
revisionKey: params.revisionKey,
|
|
3206
|
+
response,
|
|
3207
|
+
chunks
|
|
3208
|
+
}),
|
|
3209
|
+
storage: params.parsedStorageConfig.storage,
|
|
3210
|
+
documentId: params.documentId,
|
|
3211
|
+
revisionKey: params.revisionKey,
|
|
3212
|
+
source: "remoteReconstruction"
|
|
3213
|
+
});
|
|
3214
|
+
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
3215
|
+
documentId: params.documentId,
|
|
3216
|
+
revisionKey: params.revisionKey,
|
|
3217
|
+
nextChunkPage: totalPages + 1,
|
|
3218
|
+
status: "completed",
|
|
3219
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3220
|
+
});
|
|
3221
|
+
return {
|
|
3222
|
+
documentId: params.documentId,
|
|
3223
|
+
revisionKey: params.revisionKey,
|
|
3224
|
+
completed: true,
|
|
3225
|
+
commit: synced.commit
|
|
3226
|
+
};
|
|
3227
|
+
}
|
|
3228
|
+
async readRemoteReconstructionChunks(params) {
|
|
3229
|
+
const chunks = [];
|
|
3230
|
+
for (let page = 1; page <= params.totalPages; page += 1) {
|
|
3231
|
+
const object = await params.parsedStorageConfig.storage.readObject({
|
|
3232
|
+
documentId: params.documentId,
|
|
3233
|
+
revisionKey: params.revisionKey,
|
|
3234
|
+
path: getRemoteReconstructionPagePath(page)
|
|
3235
|
+
});
|
|
3236
|
+
if (!object) {
|
|
3237
|
+
return null;
|
|
3238
|
+
}
|
|
3239
|
+
const payload = readJsonObject(object);
|
|
3240
|
+
if (!isDocumentChunkArray(payload.chunks)) {
|
|
3241
|
+
return null;
|
|
3242
|
+
}
|
|
3243
|
+
chunks.push(...payload.chunks);
|
|
3244
|
+
}
|
|
3245
|
+
return chunks;
|
|
3246
|
+
}
|
|
3247
|
+
requireParsedStorageConfig() {
|
|
3248
|
+
if (!this.parsedStorageConfig) {
|
|
3249
|
+
throw new ValidationError("parsed document storage is not configured");
|
|
3250
|
+
}
|
|
3251
|
+
return this.parsedStorageConfig;
|
|
3252
|
+
}
|
|
3253
|
+
async findLocalDocumentByRemoteId(documentId) {
|
|
3254
|
+
const documents = await this.store.listDocuments();
|
|
3255
|
+
return documents.find((document) => document.documentId === documentId);
|
|
3256
|
+
}
|
|
3257
|
+
async getPublishedDocumentJobId(documentId) {
|
|
3258
|
+
const response = await this.client.documents.listChunks(documentId, {
|
|
3259
|
+
page: 1,
|
|
3260
|
+
pageSize: 1,
|
|
3261
|
+
includeAssetUrls: true
|
|
3262
|
+
});
|
|
3263
|
+
if (!response.jobId) {
|
|
3264
|
+
throw new Error(
|
|
3265
|
+
`Cannot sync server document ${documentId}: current published job id was not returned.`
|
|
3266
|
+
);
|
|
3267
|
+
}
|
|
3268
|
+
return response.jobId;
|
|
2009
3269
|
}
|
|
2010
3270
|
};
|
|
2011
3271
|
function normalizeDocumentReference(reference) {
|
|
@@ -2024,6 +3284,496 @@ function createLocalDocumentIdForRemote(documentId) {
|
|
|
2024
3284
|
const hash = createHash2("sha256").update(documentId).digest("hex").slice(0, 16);
|
|
2025
3285
|
return `remote_${hash}`;
|
|
2026
3286
|
}
|
|
3287
|
+
var defaultParsedStorageScheduler = {
|
|
3288
|
+
schedule(task) {
|
|
3289
|
+
void task();
|
|
3290
|
+
}
|
|
3291
|
+
};
|
|
3292
|
+
function getParsedStorageLimits(limits) {
|
|
3293
|
+
return {
|
|
3294
|
+
remotePageSize: clampLimit(
|
|
3295
|
+
limits?.remotePageSize,
|
|
3296
|
+
DEFAULT_REMOTE_SCAN_PAGE_SIZE,
|
|
3297
|
+
MAX_PAGE_SIZE
|
|
3298
|
+
),
|
|
3299
|
+
maxPagesPerSync: clampLimit(limits?.maxPagesPerSync, DEFAULT_MAX_SYNC_PAGES, 1e3),
|
|
3300
|
+
syncDeadlineMs: clampLimit(limits?.syncDeadlineMs, DEFAULT_SYNC_DEADLINE_MS, 6e4),
|
|
3301
|
+
grepMaxPages: clampLimit(limits?.grepMaxPages, DEFAULT_GREP_MAX_PAGES, 1e3),
|
|
3302
|
+
grepDeadlineMs: clampLimit(limits?.grepDeadlineMs, DEFAULT_GREP_DEADLINE_MS, 6e4),
|
|
3303
|
+
outlineMaxPages: clampLimit(limits?.outlineMaxPages, DEFAULT_OUTLINE_MAX_PAGES, 1e3),
|
|
3304
|
+
outlineDeadlineMs: clampLimit(limits?.outlineDeadlineMs, DEFAULT_OUTLINE_DEADLINE_MS, 6e4)
|
|
3305
|
+
};
|
|
3306
|
+
}
|
|
3307
|
+
function validateReadParams(params) {
|
|
3308
|
+
const usesPagedDisplay = params.page !== void 0 || params.pageSize !== void 0;
|
|
3309
|
+
if (!usesPagedDisplay) {
|
|
3310
|
+
return;
|
|
3311
|
+
}
|
|
3312
|
+
if (params.sectionPath !== void 0 || params.startChunk !== void 0 || params.endChunk !== void 0 || params.chunkId !== void 0) {
|
|
3313
|
+
throw new ValidationError(
|
|
3314
|
+
"page and pageSize cannot be combined with sectionPath, startChunk, endChunk, or chunkId"
|
|
3315
|
+
);
|
|
3316
|
+
}
|
|
3317
|
+
}
|
|
3318
|
+
function hasScanReadFilters(params) {
|
|
3319
|
+
return params.sectionPath !== void 0 || params.startChunk !== void 0 || params.endChunk !== void 0 || params.chunkId !== void 0;
|
|
3320
|
+
}
|
|
3321
|
+
function getReadPageParams(params) {
|
|
3322
|
+
return {
|
|
3323
|
+
page: clampLimit(params.page, 1, Number.MAX_SAFE_INTEGER),
|
|
3324
|
+
pageSize: clampLimit(params.pageSize ?? params.limit, DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE)
|
|
3325
|
+
};
|
|
3326
|
+
}
|
|
3327
|
+
function getRemoteRevisionKey(response) {
|
|
3328
|
+
const revisionKey = response.jobResultId ?? response.jobId;
|
|
3329
|
+
if (!revisionKey) {
|
|
3330
|
+
throw new Error(
|
|
3331
|
+
`Cannot read server document ${response.documentId}: current published revision was not returned.`
|
|
3332
|
+
);
|
|
3333
|
+
}
|
|
3334
|
+
return revisionKey;
|
|
3335
|
+
}
|
|
3336
|
+
function getRemoteReconstructionPagePath(page) {
|
|
3337
|
+
return `${remoteReconstructionPageDirectory}/page-${page}.json`;
|
|
3338
|
+
}
|
|
3339
|
+
function createRemoteKnowledgeDocument(params) {
|
|
3340
|
+
return {
|
|
3341
|
+
localDocumentId: params.documentId,
|
|
3342
|
+
jobId: params.response.jobId ?? params.revisionKey,
|
|
3343
|
+
documentId: params.documentId,
|
|
3344
|
+
namespace: params.response.namespace,
|
|
3345
|
+
sourceFileName: params.documentId,
|
|
3346
|
+
chunkCount: params.response.pagination.total,
|
|
3347
|
+
typeCounts: params.typeCounts ?? countRemoteChunkTypes(params.response.chunks),
|
|
3348
|
+
resultDirectoryPath: `remote:${params.documentId}`,
|
|
3349
|
+
createdAt: /* @__PURE__ */ new Date(0),
|
|
3350
|
+
updatedAt: /* @__PURE__ */ new Date(0)
|
|
3351
|
+
};
|
|
3352
|
+
}
|
|
3353
|
+
function createMinimalParseResultFromRemoteChunks(params) {
|
|
3354
|
+
const chunks = params.chunks.map(toMinimalChunkFromRemoteChunk);
|
|
3355
|
+
const typeCounts = countRemoteChunkTypes(params.chunks);
|
|
3356
|
+
return {
|
|
3357
|
+
manifest: {
|
|
3358
|
+
version: "2.0",
|
|
3359
|
+
jobId: params.response.jobId ?? params.revisionKey,
|
|
3360
|
+
sourceFileName: params.documentId,
|
|
3361
|
+
statistics: {
|
|
3362
|
+
totalChunks: chunks.length,
|
|
3363
|
+
textChunks: typeCounts.text,
|
|
3364
|
+
imageChunks: typeCounts.image,
|
|
3365
|
+
tableChunks: typeCounts.table,
|
|
3366
|
+
pageChunks: typeCounts.page
|
|
3367
|
+
}
|
|
3368
|
+
},
|
|
3369
|
+
chunks,
|
|
3370
|
+
rawZip: Buffer.from(""),
|
|
3371
|
+
namespace: params.response.namespace,
|
|
3372
|
+
documentId: params.documentId,
|
|
3373
|
+
textChunks: chunks.filter(
|
|
3374
|
+
(chunk) => chunk.type === "text"
|
|
3375
|
+
),
|
|
3376
|
+
imageChunks: chunks.filter(
|
|
3377
|
+
(chunk) => chunk.type === "image"
|
|
3378
|
+
),
|
|
3379
|
+
tableChunks: chunks.filter(
|
|
3380
|
+
(chunk) => chunk.type === "table"
|
|
3381
|
+
),
|
|
3382
|
+
pageChunks: chunks.filter(
|
|
3383
|
+
(chunk) => chunk.type === "page"
|
|
3384
|
+
),
|
|
3385
|
+
jobId: params.response.jobId ?? params.revisionKey,
|
|
3386
|
+
statistics: {
|
|
3387
|
+
totalChunks: chunks.length,
|
|
3388
|
+
textChunks: typeCounts.text,
|
|
3389
|
+
imageChunks: typeCounts.image,
|
|
3390
|
+
tableChunks: typeCounts.table,
|
|
3391
|
+
pageChunks: typeCounts.page
|
|
3392
|
+
},
|
|
3393
|
+
getChunk: (chunkId) => chunks.find((chunk) => chunk.chunkId === chunkId),
|
|
3394
|
+
save: () => Promise.reject(new Error("Minimal remote reconstruction cannot be saved directly."))
|
|
3395
|
+
};
|
|
3396
|
+
}
|
|
3397
|
+
function toMinimalChunkFromRemoteChunk(chunk) {
|
|
3398
|
+
const baseChunk = {
|
|
3399
|
+
chunkId: chunk.chunkId,
|
|
3400
|
+
contentSource: chunk.contentSource ?? void 0,
|
|
3401
|
+
content: chunk.content ?? "",
|
|
3402
|
+
path: chunk.sourceChunkPath ?? chunk.sectionPath ?? "",
|
|
3403
|
+
metadata: cloneMetadata(chunk.metadata)
|
|
3404
|
+
};
|
|
3405
|
+
if (chunk.chunkType === "image") {
|
|
3406
|
+
return {
|
|
3407
|
+
...baseChunk,
|
|
3408
|
+
type: "image",
|
|
3409
|
+
filePath: chunk.filePath ?? "",
|
|
3410
|
+
assetUrl: chunk.assetUrl ?? void 0,
|
|
3411
|
+
data: Buffer.alloc(0),
|
|
3412
|
+
format: inferContentTypeFromPath2(chunk.filePath ?? "").replace("image/", ""),
|
|
3413
|
+
save: () => Promise.resolve(chunk.filePath ?? "")
|
|
3414
|
+
};
|
|
3415
|
+
}
|
|
3416
|
+
if (chunk.chunkType === "table") {
|
|
3417
|
+
return {
|
|
3418
|
+
...baseChunk,
|
|
3419
|
+
type: "table",
|
|
3420
|
+
filePath: chunk.filePath ?? "",
|
|
3421
|
+
assetUrl: chunk.assetUrl ?? void 0,
|
|
3422
|
+
html: chunk.content ?? "",
|
|
3423
|
+
save: () => Promise.resolve(chunk.filePath ?? "")
|
|
3424
|
+
};
|
|
3425
|
+
}
|
|
3426
|
+
if (chunk.chunkType === "page") {
|
|
3427
|
+
return {
|
|
3428
|
+
...baseChunk,
|
|
3429
|
+
type: "page"
|
|
3430
|
+
};
|
|
3431
|
+
}
|
|
3432
|
+
return {
|
|
3433
|
+
...baseChunk,
|
|
3434
|
+
type: "text"
|
|
3435
|
+
};
|
|
3436
|
+
}
|
|
3437
|
+
function createStoredKnowledgeDocument(params) {
|
|
3438
|
+
return {
|
|
3439
|
+
localDocumentId: params.documentId,
|
|
3440
|
+
jobId: params.storedResult.manifest.jobId || params.revisionKey,
|
|
3441
|
+
documentId: params.documentId,
|
|
3442
|
+
sourceFileName: params.storedResult.manifest.sourceFileName,
|
|
3443
|
+
chunkCount: params.storedResult.chunks.length,
|
|
3444
|
+
typeCounts: params.storedResult.typeCounts,
|
|
3445
|
+
resultDirectoryPath: `parsed-storage:${params.documentId}`,
|
|
3446
|
+
createdAt: new Date(params.storedResult.commit.committedAt),
|
|
3447
|
+
updatedAt: new Date(params.storedResult.commit.committedAt)
|
|
3448
|
+
};
|
|
3449
|
+
}
|
|
3450
|
+
function toRemoteReadChunk(chunk) {
|
|
3451
|
+
const metadata = cloneMetadata(chunk.metadata);
|
|
3452
|
+
return {
|
|
3453
|
+
position: toChunkPosition(chunk),
|
|
3454
|
+
chunkId: chunk.chunkId,
|
|
3455
|
+
chunkType: chunk.chunkType,
|
|
3456
|
+
contentSource: chunk.contentSource ?? void 0,
|
|
3457
|
+
content: chunk.content ?? "",
|
|
3458
|
+
readableContent: getReadableRemoteChunkContent(chunk, metadata),
|
|
3459
|
+
sectionPath: chunk.sectionPath ?? "",
|
|
3460
|
+
sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? "",
|
|
3461
|
+
filePath: chunk.filePath ?? void 0,
|
|
3462
|
+
assetUrl: chunk.assetUrl ?? void 0,
|
|
3463
|
+
pageNumbers: getChunkPageNumbers(metadata),
|
|
3464
|
+
metadata
|
|
3465
|
+
};
|
|
3466
|
+
}
|
|
3467
|
+
function getReadableRemoteChunkContent(chunk, metadata) {
|
|
3468
|
+
if (chunk.chunkType !== "page") {
|
|
3469
|
+
return chunk.content ?? "";
|
|
3470
|
+
}
|
|
3471
|
+
const summary = metadata.summary;
|
|
3472
|
+
return typeof summary === "string" && summary.trim().length > 0 ? summary : chunk.content ?? "";
|
|
3473
|
+
}
|
|
3474
|
+
function toIndexedRemoteChunk(chunk) {
|
|
3475
|
+
const readChunk = toRemoteReadChunk(chunk);
|
|
3476
|
+
return toIndexedReadChunk(readChunk);
|
|
3477
|
+
}
|
|
3478
|
+
function parseStoredResultChunks(params) {
|
|
3479
|
+
const rawChunks = extractStoredChunkRecords(params.payload);
|
|
3480
|
+
return rawChunks.map((record, index) => {
|
|
3481
|
+
const metadata = readRecordMetadata(record);
|
|
3482
|
+
const chunkType = readRecordString(record, "type", "chunkType", "chunk_type");
|
|
3483
|
+
const content = readRecordString(record, "content") ?? "";
|
|
3484
|
+
const sourceChunkPath = readRecordString(record, "path", "sourceChunkPath", "source_chunk_path") ?? "";
|
|
3485
|
+
const chunk = {
|
|
3486
|
+
position: index + 1,
|
|
3487
|
+
chunkId: readRecordString(record, "chunkId", "chunk_id") ?? "",
|
|
3488
|
+
chunkType: isKnowledgeChunkType(chunkType) ? chunkType : "text",
|
|
3489
|
+
contentSource: readRecordString(record, "contentSource", "content_source"),
|
|
3490
|
+
content,
|
|
3491
|
+
readableContent: chunkType === "page" && typeof metadata.summary === "string" && metadata.summary.trim().length > 0 ? metadata.summary : content,
|
|
3492
|
+
sectionPath: normalizeSectionPath(sourceChunkPath, params.sourceFileName),
|
|
3493
|
+
sourceChunkPath,
|
|
3494
|
+
filePath: readRecordString(record, "filePath", "file_path") ?? (typeof metadata.filePath === "string" ? metadata.filePath : void 0),
|
|
3495
|
+
assetUrl: readRecordString(record, "assetUrl", "asset_url"),
|
|
3496
|
+
pageNumbers: getChunkPageNumbers(metadata),
|
|
3497
|
+
metadata
|
|
3498
|
+
};
|
|
3499
|
+
return toIndexedReadChunk(chunk);
|
|
3500
|
+
});
|
|
3501
|
+
}
|
|
3502
|
+
function extractStoredChunkRecords(payload) {
|
|
3503
|
+
if (Array.isArray(payload)) {
|
|
3504
|
+
return payload.filter(isRecord2);
|
|
3505
|
+
}
|
|
3506
|
+
if (isRecord2(payload) && Array.isArray(payload.chunks)) {
|
|
3507
|
+
return payload.chunks.filter(isRecord2);
|
|
3508
|
+
}
|
|
3509
|
+
return [];
|
|
3510
|
+
}
|
|
3511
|
+
function readRecordString(record, ...keys) {
|
|
3512
|
+
for (const key of keys) {
|
|
3513
|
+
const value = record[key];
|
|
3514
|
+
if (typeof value === "string") return value;
|
|
3515
|
+
}
|
|
3516
|
+
return void 0;
|
|
3517
|
+
}
|
|
3518
|
+
function readRecordNumber(record, ...keys) {
|
|
3519
|
+
for (const key of keys) {
|
|
3520
|
+
const value = record[key];
|
|
3521
|
+
if (typeof value === "number") return value;
|
|
3522
|
+
}
|
|
3523
|
+
return void 0;
|
|
3524
|
+
}
|
|
3525
|
+
function readRecordMetadata(record) {
|
|
3526
|
+
const metadata = record.metadata;
|
|
3527
|
+
return isRecord2(metadata) ? keysToCamel(metadata) : {};
|
|
3528
|
+
}
|
|
3529
|
+
function isKnowledgeChunkType(value) {
|
|
3530
|
+
return value === "text" || value === "image" || value === "table" || value === "page";
|
|
3531
|
+
}
|
|
3532
|
+
function toIndexedReadChunk(chunk) {
|
|
3533
|
+
return {
|
|
3534
|
+
source: createSyntheticChunk(chunk),
|
|
3535
|
+
position: chunk.position,
|
|
3536
|
+
chunkId: chunk.chunkId,
|
|
3537
|
+
chunkType: chunk.chunkType,
|
|
3538
|
+
contentSource: chunk.contentSource,
|
|
3539
|
+
content: chunk.content,
|
|
3540
|
+
readableContent: chunk.readableContent,
|
|
3541
|
+
sectionPath: chunk.sectionPath,
|
|
3542
|
+
sourceChunkPath: chunk.sourceChunkPath,
|
|
3543
|
+
filePath: chunk.filePath,
|
|
3544
|
+
assetUrl: chunk.assetUrl,
|
|
3545
|
+
pageNumbers: chunk.pageNumbers,
|
|
3546
|
+
metadata: chunk.metadata
|
|
3547
|
+
};
|
|
3548
|
+
}
|
|
3549
|
+
function createSyntheticChunk(chunk) {
|
|
3550
|
+
return {
|
|
3551
|
+
chunkId: chunk.chunkId,
|
|
3552
|
+
type: chunk.chunkType,
|
|
3553
|
+
contentSource: chunk.contentSource,
|
|
3554
|
+
content: chunk.content,
|
|
3555
|
+
path: chunk.sourceChunkPath,
|
|
3556
|
+
metadata: chunk.metadata
|
|
3557
|
+
};
|
|
3558
|
+
}
|
|
3559
|
+
function toChunkPosition(chunk) {
|
|
3560
|
+
return chunk ? chunk.sortOrder + 1 : 1;
|
|
3561
|
+
}
|
|
3562
|
+
function buildFlatSectionsFromIndexedChunks(chunks) {
|
|
3563
|
+
const byPath = /* @__PURE__ */ new Map();
|
|
3564
|
+
for (const chunk of chunks) {
|
|
3565
|
+
const path4 = chunk.sectionPath || chunk.sourceChunkPath;
|
|
3566
|
+
const existing = byPath.get(path4);
|
|
3567
|
+
if (existing) {
|
|
3568
|
+
addChunkToSection(existing, chunk);
|
|
3569
|
+
} else {
|
|
3570
|
+
byPath.set(path4, createSectionFromChunk(path4, chunk));
|
|
3571
|
+
}
|
|
3572
|
+
}
|
|
3573
|
+
return [...byPath.values()].sort(compareSections);
|
|
3574
|
+
}
|
|
3575
|
+
function matchesRemoteReadScope(chunk, params) {
|
|
3576
|
+
return matchesReadScope(toIndexedRemoteChunk(chunk), params);
|
|
3577
|
+
}
|
|
3578
|
+
function getNextChunkPosition(chunks, selected) {
|
|
3579
|
+
const lastSelected = selected[selected.length - 1];
|
|
3580
|
+
if (!lastSelected || selected.length >= chunks.length) {
|
|
3581
|
+
return void 0;
|
|
3582
|
+
}
|
|
3583
|
+
return chunks[chunks.indexOf(lastSelected) + 1]?.position;
|
|
3584
|
+
}
|
|
3585
|
+
async function applyStoredAssetUrlsToReadChunks(params) {
|
|
3586
|
+
return Promise.all(
|
|
3587
|
+
params.chunks.map(
|
|
3588
|
+
(chunk) => applyStoredAssetUrlsToReadChunk({
|
|
3589
|
+
...params,
|
|
3590
|
+
chunk
|
|
3591
|
+
})
|
|
3592
|
+
)
|
|
3593
|
+
);
|
|
3594
|
+
}
|
|
3595
|
+
async function applyStoredAssetUrlsToReadChunk(params) {
|
|
3596
|
+
const metadata = cloneMetadata(params.chunk.metadata);
|
|
3597
|
+
const storedAssetUrl = await getStoredObjectUrl({
|
|
3598
|
+
storageConfig: params.storageConfig,
|
|
3599
|
+
documentId: params.documentId,
|
|
3600
|
+
revisionKey: params.revisionKey,
|
|
3601
|
+
path: params.chunk.filePath
|
|
3602
|
+
});
|
|
3603
|
+
const pageAssets = await applyStoredPageAssetUrls({
|
|
3604
|
+
storageConfig: params.storageConfig,
|
|
3605
|
+
documentId: params.documentId,
|
|
3606
|
+
revisionKey: params.revisionKey,
|
|
3607
|
+
metadata
|
|
3608
|
+
});
|
|
3609
|
+
if (pageAssets) {
|
|
3610
|
+
metadata.pageAssets = pageAssets;
|
|
3611
|
+
delete metadata.page_assets;
|
|
3612
|
+
}
|
|
3613
|
+
return {
|
|
3614
|
+
...params.chunk,
|
|
3615
|
+
assetUrl: storedAssetUrl ?? params.chunk.assetUrl,
|
|
3616
|
+
metadata
|
|
3617
|
+
};
|
|
3618
|
+
}
|
|
3619
|
+
async function applyStoredPageAssetUrls(params) {
|
|
3620
|
+
const pageAssets = getStoredPageAssets(params.metadata);
|
|
3621
|
+
if (!pageAssets) {
|
|
3622
|
+
return void 0;
|
|
3623
|
+
}
|
|
3624
|
+
return Promise.all(
|
|
3625
|
+
pageAssets.map(async (pageAsset) => {
|
|
3626
|
+
if (!isRecord2(pageAsset)) {
|
|
3627
|
+
return pageAsset;
|
|
3628
|
+
}
|
|
3629
|
+
const artifactRef = readRecordString(pageAsset, "artifactRef", "artifact_ref");
|
|
3630
|
+
if (typeof artifactRef !== "string") {
|
|
3631
|
+
return pageAsset;
|
|
3632
|
+
}
|
|
3633
|
+
const storedAssetUrl = await getStoredObjectUrl({
|
|
3634
|
+
storageConfig: params.storageConfig,
|
|
3635
|
+
documentId: params.documentId,
|
|
3636
|
+
revisionKey: params.revisionKey,
|
|
3637
|
+
path: artifactRef
|
|
3638
|
+
});
|
|
3639
|
+
if (storedAssetUrl) {
|
|
3640
|
+
return {
|
|
3641
|
+
...pageAsset,
|
|
3642
|
+
artifactRef,
|
|
3643
|
+
assetUrl: storedAssetUrl
|
|
3644
|
+
};
|
|
3645
|
+
}
|
|
3646
|
+
return pageAsset;
|
|
3647
|
+
})
|
|
3648
|
+
);
|
|
3649
|
+
}
|
|
3650
|
+
function getStoredPageAssets(metadata) {
|
|
3651
|
+
const pageAssets = toUnknownArray(metadata.pageAssets) ?? toUnknownArray(metadata.page_assets) ?? void 0;
|
|
3652
|
+
return pageAssets?.map(normalizeStoredPageAsset);
|
|
3653
|
+
}
|
|
3654
|
+
function normalizeStoredPageAsset(pageAsset) {
|
|
3655
|
+
if (!isRecord2(pageAsset)) {
|
|
3656
|
+
return pageAsset;
|
|
3657
|
+
}
|
|
3658
|
+
const normalizedPageAsset = { ...pageAsset };
|
|
3659
|
+
delete normalizedPageAsset.page_num;
|
|
3660
|
+
delete normalizedPageAsset.artifact_ref;
|
|
3661
|
+
delete normalizedPageAsset.asset_url;
|
|
3662
|
+
delete normalizedPageAsset.content_type;
|
|
3663
|
+
const pageNum = readRecordNumber(pageAsset, "pageNum", "page_num");
|
|
3664
|
+
const artifactRef = readRecordString(pageAsset, "artifactRef", "artifact_ref");
|
|
3665
|
+
const assetUrl = readRecordString(pageAsset, "assetUrl", "asset_url");
|
|
3666
|
+
const contentType = readRecordString(pageAsset, "contentType", "content_type");
|
|
3667
|
+
return {
|
|
3668
|
+
...normalizedPageAsset,
|
|
3669
|
+
...pageNum === void 0 ? {} : { pageNum },
|
|
3670
|
+
...artifactRef === void 0 ? {} : { artifactRef },
|
|
3671
|
+
...assetUrl === void 0 ? {} : { assetUrl },
|
|
3672
|
+
...contentType === void 0 ? {} : { contentType }
|
|
3673
|
+
};
|
|
3674
|
+
}
|
|
3675
|
+
async function getStoredObjectUrl(params) {
|
|
3676
|
+
if (!params.path) {
|
|
3677
|
+
return void 0;
|
|
3678
|
+
}
|
|
3679
|
+
return await params.storageConfig.storage.getObjectUrl?.({
|
|
3680
|
+
documentId: params.documentId,
|
|
3681
|
+
revisionKey: params.revisionKey,
|
|
3682
|
+
path: params.path
|
|
3683
|
+
}) ?? void 0;
|
|
3684
|
+
}
|
|
3685
|
+
function countRemoteChunkTypes(chunks) {
|
|
3686
|
+
return chunks.reduce(
|
|
3687
|
+
(counts, chunk) => {
|
|
3688
|
+
counts[chunk.chunkType] += 1;
|
|
3689
|
+
return counts;
|
|
3690
|
+
},
|
|
3691
|
+
{ text: 0, image: 0, table: 0, page: 0 }
|
|
3692
|
+
);
|
|
3693
|
+
}
|
|
3694
|
+
function createContinuationCursor(cursor) {
|
|
3695
|
+
return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
|
|
3696
|
+
}
|
|
3697
|
+
function createGrepContinuationCursor(params) {
|
|
3698
|
+
if (params.matchIndex + 1 < params.chunkMatchCount) {
|
|
3699
|
+
return createContinuationCursor({
|
|
3700
|
+
documentId: params.documentId,
|
|
3701
|
+
revisionKey: params.revisionKey,
|
|
3702
|
+
nextPage: params.response.pagination.page,
|
|
3703
|
+
nextChunkIndex: params.chunkIndex,
|
|
3704
|
+
nextMatchIndex: params.matchIndex + 1
|
|
3705
|
+
});
|
|
3706
|
+
}
|
|
3707
|
+
if (params.chunkIndex + 1 < params.response.chunks.length) {
|
|
3708
|
+
return createContinuationCursor({
|
|
3709
|
+
documentId: params.documentId,
|
|
3710
|
+
revisionKey: params.revisionKey,
|
|
3711
|
+
nextPage: params.response.pagination.page,
|
|
3712
|
+
nextChunkIndex: params.chunkIndex + 1,
|
|
3713
|
+
nextMatchIndex: 0
|
|
3714
|
+
});
|
|
3715
|
+
}
|
|
3716
|
+
if (params.response.pagination.page < params.response.pagination.totalPages) {
|
|
3717
|
+
return createContinuationCursor({
|
|
3718
|
+
documentId: params.documentId,
|
|
3719
|
+
revisionKey: params.revisionKey,
|
|
3720
|
+
nextPage: params.response.pagination.page + 1,
|
|
3721
|
+
nextChunkIndex: 0,
|
|
3722
|
+
nextMatchIndex: 0
|
|
3723
|
+
});
|
|
3724
|
+
}
|
|
3725
|
+
return void 0;
|
|
3726
|
+
}
|
|
3727
|
+
function parseContinuationCursor(value) {
|
|
3728
|
+
if (!value) {
|
|
3729
|
+
return null;
|
|
3730
|
+
}
|
|
3731
|
+
try {
|
|
3732
|
+
const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
3733
|
+
if (!isRecord2(parsed)) {
|
|
3734
|
+
return null;
|
|
3735
|
+
}
|
|
3736
|
+
const documentId = parsed.documentId;
|
|
3737
|
+
const revisionKey = parsed.revisionKey;
|
|
3738
|
+
const nextPage = parsed.nextPage;
|
|
3739
|
+
const nextChunkIndex = parsed.nextChunkIndex;
|
|
3740
|
+
const nextMatchIndex = parsed.nextMatchIndex;
|
|
3741
|
+
if (typeof documentId !== "string" || typeof revisionKey !== "string" || typeof nextPage !== "number" || !Number.isInteger(nextPage) || nextPage < 1 || !isOptionalNonNegativeInteger(nextChunkIndex) || !isOptionalNonNegativeInteger(nextMatchIndex)) {
|
|
3742
|
+
return null;
|
|
3743
|
+
}
|
|
3744
|
+
return {
|
|
3745
|
+
documentId,
|
|
3746
|
+
revisionKey,
|
|
3747
|
+
nextPage,
|
|
3748
|
+
nextChunkIndex,
|
|
3749
|
+
nextMatchIndex
|
|
3750
|
+
};
|
|
3751
|
+
} catch {
|
|
3752
|
+
return null;
|
|
3753
|
+
}
|
|
3754
|
+
}
|
|
3755
|
+
function isOptionalNonNegativeInteger(value) {
|
|
3756
|
+
return value === void 0 || typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
3757
|
+
}
|
|
3758
|
+
function cloneMetadata(metadata) {
|
|
3759
|
+
return { ...metadata };
|
|
3760
|
+
}
|
|
3761
|
+
function toUnknownArray(value) {
|
|
3762
|
+
return Array.isArray(value) ? value : null;
|
|
3763
|
+
}
|
|
3764
|
+
function inferContentTypeFromPath2(filePath) {
|
|
3765
|
+
const extension = filePath.split(".").at(-1)?.toLowerCase();
|
|
3766
|
+
if (extension === "png") return "image/png";
|
|
3767
|
+
if (extension === "jpg" || extension === "jpeg") return "image/jpeg";
|
|
3768
|
+
if (extension === "html" || extension === "htm") return "text/html; charset=utf-8";
|
|
3769
|
+
return "application/octet-stream";
|
|
3770
|
+
}
|
|
3771
|
+
function isRecord2(value) {
|
|
3772
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3773
|
+
}
|
|
3774
|
+
function isDocumentChunkArray(value) {
|
|
3775
|
+
return Array.isArray(value) && value.every(isRecord2);
|
|
3776
|
+
}
|
|
2027
3777
|
function indexChunks(result) {
|
|
2028
3778
|
return result.chunks.map((chunk, index) => {
|
|
2029
3779
|
const filePath = getChunkFilePath2(chunk);
|
|
@@ -2039,6 +3789,7 @@ function indexChunks(result) {
|
|
|
2039
3789
|
sectionPath: normalizeSectionPath(chunk.path, result.manifest.sourceFileName),
|
|
2040
3790
|
sourceChunkPath: chunk.path,
|
|
2041
3791
|
filePath,
|
|
3792
|
+
assetUrl: getChunkAssetUrl(chunk),
|
|
2042
3793
|
pageNumbers: getChunkPageNumbers(chunk.metadata),
|
|
2043
3794
|
metadata: chunk.metadata
|
|
2044
3795
|
};
|
|
@@ -2052,12 +3803,11 @@ function getReadableChunkContent(chunk) {
|
|
|
2052
3803
|
return typeof summary === "string" && summary.trim().length > 0 ? summary : chunk.content;
|
|
2053
3804
|
}
|
|
2054
3805
|
function getChunkPageNumbers(metadata) {
|
|
2055
|
-
const
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
(pageNumber) => Number.isInteger(pageNumber) && pageNumber > 0
|
|
3806
|
+
const values = [metadata.pageNums, metadata.page_nums].filter(Array.isArray);
|
|
3807
|
+
const pageNumbers = values.flatMap(
|
|
3808
|
+
(value) => value.filter(
|
|
3809
|
+
(pageNumber) => Number.isInteger(pageNumber) && pageNumber > 0
|
|
3810
|
+
)
|
|
2061
3811
|
);
|
|
2062
3812
|
return pageNumbers.length > 0 ? [...new Set(pageNumbers)].sort((left, right) => left - right) : void 0;
|
|
2063
3813
|
}
|
|
@@ -2068,14 +3818,17 @@ function getChunkFilePath2(chunk) {
|
|
|
2068
3818
|
const filePath = chunk.metadata.filePath;
|
|
2069
3819
|
return typeof filePath === "string" ? filePath : void 0;
|
|
2070
3820
|
}
|
|
2071
|
-
function
|
|
2072
|
-
|
|
3821
|
+
function getChunkAssetUrl(chunk) {
|
|
3822
|
+
return chunk.type === "image" || chunk.type === "table" ? chunk.assetUrl : void 0;
|
|
3823
|
+
}
|
|
3824
|
+
function normalizeSectionPath(path4, sourceFileName) {
|
|
3825
|
+
if (!path4) {
|
|
2073
3826
|
return "";
|
|
2074
3827
|
}
|
|
2075
|
-
if (
|
|
2076
|
-
return
|
|
3828
|
+
if (path4.startsWith("images/") || path4.startsWith("tables/")) {
|
|
3829
|
+
return path4;
|
|
2077
3830
|
}
|
|
2078
|
-
const parts =
|
|
3831
|
+
const parts = path4.split("/").filter(Boolean);
|
|
2079
3832
|
if (sourceFileName) {
|
|
2080
3833
|
const fileNameIndex = parts.indexOf(sourceFileName);
|
|
2081
3834
|
if (fileNameIndex >= 0) {
|
|
@@ -2097,12 +3850,12 @@ function buildFlatSections(result, chunks) {
|
|
|
2097
3850
|
}
|
|
2098
3851
|
const byPath = /* @__PURE__ */ new Map();
|
|
2099
3852
|
for (const chunk of chunks) {
|
|
2100
|
-
const
|
|
2101
|
-
const existing = byPath.get(
|
|
3853
|
+
const path4 = chunk.sectionPath || chunk.sourceChunkPath;
|
|
3854
|
+
const existing = byPath.get(path4);
|
|
2102
3855
|
if (existing) {
|
|
2103
3856
|
addChunkToSection(existing, chunk);
|
|
2104
3857
|
} else {
|
|
2105
|
-
byPath.set(
|
|
3858
|
+
byPath.set(path4, createSectionFromChunk(path4, chunk));
|
|
2106
3859
|
}
|
|
2107
3860
|
}
|
|
2108
3861
|
return [...byPath.values()].sort(compareSections);
|
|
@@ -2236,6 +3989,7 @@ function toReadChunk(chunk) {
|
|
|
2236
3989
|
sectionPath: chunk.sectionPath,
|
|
2237
3990
|
sourceChunkPath: chunk.sourceChunkPath,
|
|
2238
3991
|
filePath: chunk.filePath,
|
|
3992
|
+
assetUrl: chunk.assetUrl,
|
|
2239
3993
|
pageNumbers: chunk.pageNumbers,
|
|
2240
3994
|
metadata: chunk.metadata
|
|
2241
3995
|
};
|
|
@@ -2412,7 +4166,12 @@ var Knowhere = class {
|
|
|
2412
4166
|
const result = await this.jobs.load(jobResult, {
|
|
2413
4167
|
verifyChecksum: params.verifyChecksum
|
|
2414
4168
|
});
|
|
2415
|
-
|
|
4169
|
+
const enrichedResult = enrichParseResult(result, {
|
|
4170
|
+
namespace: jobResult.namespace,
|
|
4171
|
+
documentId: jobResult.documentId ?? params.documentId
|
|
4172
|
+
});
|
|
4173
|
+
const storedResult = await storeParseResultAssets(enrichedResult, params.storageAdapter);
|
|
4174
|
+
return storedResult.result;
|
|
2416
4175
|
}
|
|
2417
4176
|
/**
|
|
2418
4177
|
* Start a parse job and return immediately after the URL job is created or
|
|
@@ -2455,12 +4214,167 @@ var Knowhere = class {
|
|
|
2455
4214
|
return job;
|
|
2456
4215
|
}
|
|
2457
4216
|
};
|
|
4217
|
+
|
|
4218
|
+
// src/storage/disk-parsed-document-storage.ts
|
|
4219
|
+
import { createHash as createHash3 } from "crypto";
|
|
4220
|
+
import { promises as fs4 } from "fs";
|
|
4221
|
+
import path3 from "path";
|
|
4222
|
+
var syncProgressPath = ".knowhere-sdk/sync-progress.json";
|
|
4223
|
+
var DiskParsedDocumentStorage = class {
|
|
4224
|
+
rootDirectory;
|
|
4225
|
+
constructor(rootDirectory) {
|
|
4226
|
+
this.rootDirectory = rootDirectory;
|
|
4227
|
+
}
|
|
4228
|
+
async readObject(params) {
|
|
4229
|
+
const objectPath = this.getObjectPath(params);
|
|
4230
|
+
try {
|
|
4231
|
+
const body = await fs4.readFile(objectPath);
|
|
4232
|
+
const metadata = await this.readObjectMetadata(objectPath);
|
|
4233
|
+
return {
|
|
4234
|
+
...params,
|
|
4235
|
+
body,
|
|
4236
|
+
contentType: metadata?.contentType,
|
|
4237
|
+
contentLength: body.byteLength,
|
|
4238
|
+
metadata: metadata?.metadata,
|
|
4239
|
+
url: toFileUrl(objectPath)
|
|
4240
|
+
};
|
|
4241
|
+
} catch (error) {
|
|
4242
|
+
if (isMissingFileError3(error)) {
|
|
4243
|
+
return null;
|
|
4244
|
+
}
|
|
4245
|
+
throw error;
|
|
4246
|
+
}
|
|
4247
|
+
}
|
|
4248
|
+
async writeObject(params) {
|
|
4249
|
+
const objectPath = this.getObjectPath(params);
|
|
4250
|
+
await fs4.mkdir(path3.dirname(objectPath), { recursive: true });
|
|
4251
|
+
await fs4.writeFile(objectPath, params.body);
|
|
4252
|
+
await this.writeObjectMetadata(objectPath, {
|
|
4253
|
+
contentType: params.contentType,
|
|
4254
|
+
metadata: params.metadata
|
|
4255
|
+
});
|
|
4256
|
+
return {
|
|
4257
|
+
documentId: params.documentId,
|
|
4258
|
+
revisionKey: params.revisionKey,
|
|
4259
|
+
path: params.path,
|
|
4260
|
+
url: toFileUrl(objectPath)
|
|
4261
|
+
};
|
|
4262
|
+
}
|
|
4263
|
+
async headObject(params) {
|
|
4264
|
+
const objectPath = this.getObjectPath(params);
|
|
4265
|
+
try {
|
|
4266
|
+
const stat = await fs4.stat(objectPath);
|
|
4267
|
+
const metadata = await this.readObjectMetadata(objectPath);
|
|
4268
|
+
return {
|
|
4269
|
+
...params,
|
|
4270
|
+
contentType: metadata?.contentType,
|
|
4271
|
+
contentLength: stat.size,
|
|
4272
|
+
metadata: metadata?.metadata,
|
|
4273
|
+
url: toFileUrl(objectPath)
|
|
4274
|
+
};
|
|
4275
|
+
} catch (error) {
|
|
4276
|
+
if (isMissingFileError3(error)) {
|
|
4277
|
+
return null;
|
|
4278
|
+
}
|
|
4279
|
+
throw error;
|
|
4280
|
+
}
|
|
4281
|
+
}
|
|
4282
|
+
async getObjectUrl(params) {
|
|
4283
|
+
const head = await this.headObject(params);
|
|
4284
|
+
return head?.url ?? null;
|
|
4285
|
+
}
|
|
4286
|
+
async deletePrefix(params) {
|
|
4287
|
+
await fs4.rm(this.getRevisionDirectory(params.documentId, params.revisionKey), {
|
|
4288
|
+
recursive: true,
|
|
4289
|
+
force: true
|
|
4290
|
+
});
|
|
4291
|
+
}
|
|
4292
|
+
async readSyncProgress(params) {
|
|
4293
|
+
return this.readOptionalJson({
|
|
4294
|
+
...params,
|
|
4295
|
+
path: syncProgressPath
|
|
4296
|
+
});
|
|
4297
|
+
}
|
|
4298
|
+
async writeSyncProgress(params) {
|
|
4299
|
+
await this.writeObject({
|
|
4300
|
+
documentId: params.documentId,
|
|
4301
|
+
revisionKey: params.revisionKey,
|
|
4302
|
+
path: syncProgressPath,
|
|
4303
|
+
body: Buffer.from(JSON.stringify(params, null, 2), "utf8"),
|
|
4304
|
+
contentType: "application/json; charset=utf-8"
|
|
4305
|
+
});
|
|
4306
|
+
}
|
|
4307
|
+
getObjectPath(params) {
|
|
4308
|
+
const revisionDirectory = this.getRevisionDirectory(params.documentId, params.revisionKey);
|
|
4309
|
+
const objectPath = path3.resolve(revisionDirectory, normalizeRelativeStoragePath(params.path));
|
|
4310
|
+
if (!isPathInsideDirectory2(objectPath, revisionDirectory)) {
|
|
4311
|
+
throw new Error(`Parsed document object path resolves outside storage: ${params.path}`);
|
|
4312
|
+
}
|
|
4313
|
+
return objectPath;
|
|
4314
|
+
}
|
|
4315
|
+
getRevisionDirectory(documentId, revisionKey) {
|
|
4316
|
+
return path3.join(this.rootDirectory, hashPathPart(documentId), hashPathPart(revisionKey));
|
|
4317
|
+
}
|
|
4318
|
+
async readOptionalJson(params) {
|
|
4319
|
+
const object = await this.readObject(params);
|
|
4320
|
+
if (!object) {
|
|
4321
|
+
return null;
|
|
4322
|
+
}
|
|
4323
|
+
return JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
4324
|
+
}
|
|
4325
|
+
async readObjectMetadata(filePath) {
|
|
4326
|
+
try {
|
|
4327
|
+
return JSON.parse(
|
|
4328
|
+
await fs4.readFile(getMetadataPath(filePath), "utf8")
|
|
4329
|
+
);
|
|
4330
|
+
} catch (error) {
|
|
4331
|
+
if (isMissingFileError3(error)) {
|
|
4332
|
+
return null;
|
|
4333
|
+
}
|
|
4334
|
+
throw error;
|
|
4335
|
+
}
|
|
4336
|
+
}
|
|
4337
|
+
async writeObjectMetadata(filePath, metadata) {
|
|
4338
|
+
await fs4.writeFile(getMetadataPath(filePath), JSON.stringify(metadata, null, 2), {
|
|
4339
|
+
encoding: "utf8",
|
|
4340
|
+
flag: "w"
|
|
4341
|
+
});
|
|
4342
|
+
}
|
|
4343
|
+
};
|
|
4344
|
+
function getMetadataPath(filePath) {
|
|
4345
|
+
return `${filePath}.metadata.json`;
|
|
4346
|
+
}
|
|
4347
|
+
function hashPathPart(value) {
|
|
4348
|
+
const normalized = value.trim();
|
|
4349
|
+
if (normalized.length === 0) {
|
|
4350
|
+
throw new Error("Parsed document storage path part must be a non-empty value.");
|
|
4351
|
+
}
|
|
4352
|
+
return createHash3("sha256").update(normalized).digest("hex").slice(0, 24);
|
|
4353
|
+
}
|
|
4354
|
+
function normalizeRelativeStoragePath(value) {
|
|
4355
|
+
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
4356
|
+
if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
|
|
4357
|
+
throw new Error(`Invalid parsed storage path: ${value}`);
|
|
4358
|
+
}
|
|
4359
|
+
return normalized;
|
|
4360
|
+
}
|
|
4361
|
+
function isPathInsideDirectory2(filePath, directory) {
|
|
4362
|
+
const relative = path3.relative(directory, filePath);
|
|
4363
|
+
return relative === "" || !relative.startsWith("..") && !path3.isAbsolute(relative);
|
|
4364
|
+
}
|
|
4365
|
+
function toFileUrl(filePath) {
|
|
4366
|
+
return `file://${filePath}`;
|
|
4367
|
+
}
|
|
4368
|
+
function isMissingFileError3(error) {
|
|
4369
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
4370
|
+
}
|
|
2458
4371
|
export {
|
|
2459
4372
|
APIError,
|
|
2460
4373
|
AuthenticationError,
|
|
2461
4374
|
BadRequestError,
|
|
2462
4375
|
ChecksumError,
|
|
2463
4376
|
ConflictError,
|
|
4377
|
+
DiskParsedDocumentStorage,
|
|
2464
4378
|
Documents,
|
|
2465
4379
|
GatewayTimeoutError,
|
|
2466
4380
|
InternalServerError,
|
|
@@ -2482,5 +4396,7 @@ export {
|
|
|
2482
4396
|
TimeoutError,
|
|
2483
4397
|
VERSION,
|
|
2484
4398
|
ValidationError,
|
|
2485
|
-
Knowhere as default
|
|
4399
|
+
Knowhere as default,
|
|
4400
|
+
storeParseResultAssets,
|
|
4401
|
+
syncParseResultToParsedDocumentStorage
|
|
2486
4402
|
};
|