@ontos-ai/knowhere-sdk 2.0.0 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -309,8 +309,8 @@ function enrichParseResult(parseResult2, scope) {
309
309
  }
310
310
  return parseResult2;
311
311
  }
312
- function sanitizePath(path3) {
313
- let sanitized = path3.replace(/^\/+/, "");
312
+ function sanitizePath(path4) {
313
+ let sanitized = path4.replace(/^\/+/, "");
314
314
  sanitized = sanitized.replace(/\.\.(\/|\\)/g, "");
315
315
  sanitized = sanitized.replace(/\\/g, "/");
316
316
  return sanitized;
@@ -696,8 +696,8 @@ var BaseResource = class {
696
696
  constructor(httpClient) {
697
697
  this.httpClient = httpClient;
698
698
  }
699
- endpoint(path3) {
700
- return `/v2${path3.startsWith("/") ? path3 : `/${path3}`}`;
699
+ endpoint(path4) {
700
+ return `/v2${path4.startsWith("/") ? path4 : `/${path4}`}`;
701
701
  }
702
702
  };
703
703
 
@@ -913,17 +913,28 @@ async function saveExpandedParseResult(result, directory) {
913
913
  if (result.rawZip.length > 0) {
914
914
  const didExtractZip = await tryExtractRawZip(result.rawZip, directory);
915
915
  if (didExtractZip) {
916
+ await writeExpandedParseResultMetadata(result, directory);
916
917
  return directory;
917
918
  }
918
919
  }
919
920
  await fs2.mkdir(directory, { recursive: true });
921
+ await writeExpandedParseResultMetadata(result, directory);
922
+ for (const imageChunk of result.imageChunks) {
923
+ await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data);
924
+ }
925
+ for (const tableChunk of result.tableChunks) {
926
+ await writeTextAsset(directory, tableChunk.filePath, tableChunk.html);
927
+ }
928
+ return directory;
929
+ }
930
+ async function writeExpandedParseResultMetadata(result, directory) {
920
931
  await fs2.writeFile(join(directory, "manifest.json"), JSON.stringify(result.manifest, null, 2));
921
932
  if (result.docNav) {
922
933
  await fs2.writeFile(join(directory, "doc_nav.json"), JSON.stringify(result.docNav, null, 2));
923
934
  }
924
935
  await fs2.writeFile(
925
936
  join(directory, "chunks.json"),
926
- JSON.stringify(serializeChunks(result.chunks), null, 2)
937
+ JSON.stringify(keysToSnake(serializeChunks(result.chunks)), null, 2)
927
938
  );
928
939
  if (result.chunksSlim) {
929
940
  await fs2.writeFile(
@@ -952,13 +963,6 @@ async function saveExpandedParseResult(result, directory) {
952
963
  if (result.hierarchyViewHtml) {
953
964
  await fs2.writeFile(join(directory, "hierarchy_view.html"), result.hierarchyViewHtml);
954
965
  }
955
- for (const imageChunk of result.imageChunks) {
956
- await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data);
957
- }
958
- for (const tableChunk of result.tableChunks) {
959
- await writeTextAsset(directory, tableChunk.filePath, tableChunk.html);
960
- }
961
- return directory;
962
966
  }
963
967
  async function tryExtractRawZip(zipBuffer, directory) {
964
968
  try {
@@ -1097,13 +1101,14 @@ function buildTextChunk(chunkData) {
1097
1101
  };
1098
1102
  }
1099
1103
  function buildPageChunk(chunkData) {
1104
+ const metadata = chunkData.metadata ?? {};
1100
1105
  return {
1101
1106
  chunkId: chunkData.chunkId ?? "",
1102
1107
  type: "page",
1103
1108
  contentSource: chunkData.contentSource ?? "summary",
1104
1109
  content: chunkData.content ?? "",
1105
1110
  path: chunkData.path ?? "",
1106
- metadata: chunkData.metadata ?? {}
1111
+ metadata
1107
1112
  };
1108
1113
  }
1109
1114
  function buildImageChunk(chunkData, filePath, imageBuffer) {
@@ -1114,6 +1119,7 @@ function buildImageChunk(chunkData, filePath, imageBuffer) {
1114
1119
  content: chunkData.content ?? "",
1115
1120
  path: chunkData.path ?? "",
1116
1121
  filePath,
1122
+ assetUrl: chunkData.assetUrl,
1117
1123
  data: imageBuffer,
1118
1124
  metadata: chunkData.metadata ?? {},
1119
1125
  get format() {
@@ -1132,6 +1138,7 @@ function buildTableChunk(chunkData, filePath, html) {
1132
1138
  content: chunkData.content ?? "",
1133
1139
  path: chunkData.path ?? "",
1134
1140
  filePath,
1141
+ assetUrl: chunkData.assetUrl,
1135
1142
  html,
1136
1143
  metadata: chunkData.metadata ?? {},
1137
1144
  async save(directory) {
@@ -1226,6 +1233,7 @@ function serializeChunks(chunks) {
1226
1233
  };
1227
1234
  if (chunk.type === "image" || chunk.type === "table") {
1228
1235
  rawChunk.filePath = chunk.filePath;
1236
+ rawChunk.assetUrl = chunk.assetUrl;
1229
1237
  }
1230
1238
  return rawChunk;
1231
1239
  })
@@ -1434,6 +1442,14 @@ var Documents = class extends BaseResource {
1434
1442
  this.createChunkGetRequestConfig(params)
1435
1443
  );
1436
1444
  }
1445
+ /**
1446
+ * Get a signed normalized source-file URL for compatibility and debugging.
1447
+ */
1448
+ async getPageCitationSource(documentId) {
1449
+ return this.httpClient.get(
1450
+ this.endpoint(`/documents/${documentId}/files/page-citation-source`)
1451
+ );
1452
+ }
1437
1453
  /**
1438
1454
  * Archive one canonical document by ID.
1439
1455
  */
@@ -1506,6 +1522,477 @@ var Documents = class extends BaseResource {
1506
1522
  // src/knowledge/knowledge.ts
1507
1523
  import { createHash as createHash2 } from "crypto";
1508
1524
 
1525
+ // src/storage/asset-storage.ts
1526
+ import JSZip2 from "jszip";
1527
+ var pageCitationAssetsMetadataKey = "pageAssets";
1528
+ var defaultImageContentType = "application/octet-stream";
1529
+ var tableContentType = "text/html; charset=utf-8";
1530
+ async function storeParseResultAssets(result, options) {
1531
+ if (!options) {
1532
+ return {
1533
+ result,
1534
+ assetUrlsByFilePath: {}
1535
+ };
1536
+ }
1537
+ const normalizedKeyPrefix = normalizeStorageKeyPrefix(options.keyPrefix);
1538
+ const assets = dedupeStorageAssets(await collectStorageAssets(result, normalizedKeyPrefix));
1539
+ const assetUrlsByFilePath = {};
1540
+ for (const asset of assets) {
1541
+ const storedUrl = await storeAsset(asset, options.adapter, options.skipExisting ?? true);
1542
+ if (storedUrl) {
1543
+ assetUrlsByFilePath[asset.sourcePath] = storedUrl;
1544
+ }
1545
+ }
1546
+ const normalizedResult = Object.keys(assetUrlsByFilePath).length === 0 ? result : rewriteResultAssetUrls(result, assetUrlsByFilePath);
1547
+ return {
1548
+ result: normalizedResult,
1549
+ assetUrlsByFilePath
1550
+ };
1551
+ }
1552
+ async function collectStorageAssets(result, keyPrefix) {
1553
+ const resultZip = await loadResultZipForPageCitationAssets(result);
1554
+ return [
1555
+ ...result.imageChunks.flatMap((chunk) => collectImageStorageAsset(chunk, keyPrefix)),
1556
+ ...result.tableChunks.flatMap((chunk) => collectTableStorageAsset(chunk, keyPrefix)),
1557
+ ...resultZip ? await collectPageCitationStorageAssets(result.pageChunks, keyPrefix, resultZip) : []
1558
+ ];
1559
+ }
1560
+ function collectImageStorageAsset(chunk, keyPrefix) {
1561
+ const sourcePath = normalizeResultAssetPath(chunk.filePath);
1562
+ if (!sourcePath) return [];
1563
+ return [
1564
+ {
1565
+ sourcePath,
1566
+ storageKey: `${keyPrefix}/${sourcePath}`,
1567
+ body: chunk.data,
1568
+ contentType: getImageContentType(chunk.filePath),
1569
+ metadata: {
1570
+ chunkId: chunk.chunkId,
1571
+ chunkType: chunk.type,
1572
+ sourcePath
1573
+ }
1574
+ }
1575
+ ];
1576
+ }
1577
+ function collectTableStorageAsset(chunk, keyPrefix) {
1578
+ const sourcePath = normalizeResultAssetPath(chunk.filePath);
1579
+ if (!sourcePath) return [];
1580
+ return [
1581
+ {
1582
+ sourcePath,
1583
+ storageKey: `${keyPrefix}/${sourcePath}`,
1584
+ body: Buffer.from(chunk.html, "utf8"),
1585
+ contentType: tableContentType,
1586
+ metadata: {
1587
+ chunkId: chunk.chunkId,
1588
+ chunkType: chunk.type,
1589
+ sourcePath
1590
+ }
1591
+ }
1592
+ ];
1593
+ }
1594
+ async function collectPageCitationStorageAssets(chunks, keyPrefix, resultZip) {
1595
+ const assets = [];
1596
+ for (const chunk of chunks) {
1597
+ const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]);
1598
+ for (const pageAsset of pageAssets) {
1599
+ const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef);
1600
+ if (!sourcePath) continue;
1601
+ const zipEntry = resultZip.file(sourcePath);
1602
+ if (!zipEntry) continue;
1603
+ assets.push({
1604
+ sourcePath,
1605
+ storageKey: `${keyPrefix}/${sourcePath}`,
1606
+ body: await zipEntry.async("uint8array"),
1607
+ contentType: pageAsset.contentType,
1608
+ metadata: getPageCitationAssetMetadata(chunk, pageAsset, sourcePath)
1609
+ });
1610
+ }
1611
+ }
1612
+ return assets;
1613
+ }
1614
+ function getPageCitationAssetMetadata(chunk, pageAsset, sourcePath) {
1615
+ return {
1616
+ chunkId: chunk.chunkId,
1617
+ chunkType: chunk.type,
1618
+ pageNum: String(pageAsset.pageNum),
1619
+ sourcePath
1620
+ };
1621
+ }
1622
+ async function loadResultZipForPageCitationAssets(result) {
1623
+ if (!hasPageCitationAssetReferences(result) || result.rawZip.length === 0) {
1624
+ return null;
1625
+ }
1626
+ try {
1627
+ return await JSZip2.loadAsync(result.rawZip);
1628
+ } catch (cause) {
1629
+ throw new Error("Unable to read page citation assets from the Knowhere result ZIP.", {
1630
+ cause
1631
+ });
1632
+ }
1633
+ }
1634
+ function hasPageCitationAssetReferences(result) {
1635
+ return result.pageChunks.some(
1636
+ (chunk) => parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]).length > 0
1637
+ );
1638
+ }
1639
+ function dedupeStorageAssets(assets) {
1640
+ const assetsBySourcePath = /* @__PURE__ */ new Map();
1641
+ for (const asset of assets) {
1642
+ if (!assetsBySourcePath.has(asset.sourcePath)) {
1643
+ assetsBySourcePath.set(asset.sourcePath, asset);
1644
+ }
1645
+ }
1646
+ return [...assetsBySourcePath.values()];
1647
+ }
1648
+ async function storeAsset(asset, adapter, skipExisting) {
1649
+ if (skipExisting && adapter.headObject) {
1650
+ const existing = await adapter.headObject(asset.storageKey);
1651
+ if (existing) {
1652
+ const existingUrl = existing.url ?? await adapter.getObjectUrl?.(asset.storageKey);
1653
+ if (existingUrl) return existingUrl;
1654
+ }
1655
+ }
1656
+ const writeResult = await adapter.writeObject({
1657
+ key: asset.storageKey,
1658
+ body: asset.body,
1659
+ contentType: asset.contentType,
1660
+ metadata: asset.metadata
1661
+ });
1662
+ return writeResult.url ?? await adapter.getObjectUrl?.(writeResult.key) ?? null;
1663
+ }
1664
+ function rewriteResultAssetUrls(result, assetUrlsByFilePath) {
1665
+ const chunks = result.chunks.map((chunk) => rewriteChunkAssetUrls(chunk, assetUrlsByFilePath));
1666
+ result.chunks.splice(0, result.chunks.length, ...chunks);
1667
+ refreshStaticChunkCollections(result);
1668
+ return result;
1669
+ }
1670
+ function rewriteChunkAssetUrls(chunk, assetUrlsByPath) {
1671
+ if (chunk.type === "image" || chunk.type === "table") {
1672
+ const sourcePath = normalizeResultAssetPath(chunk.filePath);
1673
+ const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : void 0;
1674
+ if (!assetUrl) return chunk;
1675
+ return {
1676
+ ...chunk,
1677
+ assetUrl
1678
+ };
1679
+ }
1680
+ if (chunk.type === "page") {
1681
+ const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]);
1682
+ if (pageAssets.length === 0) return chunk;
1683
+ const rewrittenPageAssets = pageAssets.map((pageAsset) => {
1684
+ const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef);
1685
+ const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : void 0;
1686
+ return assetUrl ? { ...pageAsset, assetUrl } : pageAsset;
1687
+ });
1688
+ return {
1689
+ ...chunk,
1690
+ metadata: {
1691
+ ...chunk.metadata,
1692
+ [pageCitationAssetsMetadataKey]: rewrittenPageAssets
1693
+ }
1694
+ };
1695
+ }
1696
+ return chunk;
1697
+ }
1698
+ function parsePageCitationAssets(value) {
1699
+ if (!Array.isArray(value)) return [];
1700
+ return value.flatMap((item) => {
1701
+ if (!isRecord(item)) return [];
1702
+ const pageNum = item.pageNum;
1703
+ const artifactRef = item.artifactRef;
1704
+ const contentType = item.contentType;
1705
+ const source = item.source;
1706
+ if (typeof pageNum !== "number" || !Number.isSafeInteger(pageNum) || pageNum <= 0) return [];
1707
+ if (typeof artifactRef !== "string") return [];
1708
+ if (contentType !== "image/png" && contentType !== "image/jpeg") return [];
1709
+ if (source !== "knowhere-rendered-page-citation-source") return [];
1710
+ return [
1711
+ {
1712
+ pageNum,
1713
+ artifactRef,
1714
+ assetUrl: typeof item.assetUrl === "string" ? item.assetUrl : void 0,
1715
+ contentType,
1716
+ width: typeof item.width === "number" ? item.width : void 0,
1717
+ height: typeof item.height === "number" ? item.height : void 0,
1718
+ source
1719
+ }
1720
+ ];
1721
+ });
1722
+ }
1723
+ function normalizeStorageKeyPrefix(value) {
1724
+ const normalized = normalizeStoragePath(value, { allowDotPrefix: false });
1725
+ if (!normalized) {
1726
+ throw new Error("storageAdapter keyPrefix must be a non-empty relative POSIX path.");
1727
+ }
1728
+ return normalized;
1729
+ }
1730
+ function normalizeResultAssetPath(value) {
1731
+ return normalizeStoragePath(value, { allowDotPrefix: true });
1732
+ }
1733
+ function normalizeStoragePath(value, options) {
1734
+ if (!value) return null;
1735
+ const trimmed = value.trim();
1736
+ if (trimmed.length === 0 || trimmed.includes("\0")) return null;
1737
+ if (trimmed.startsWith("/") || /^[A-Za-z]:/.test(trimmed)) return null;
1738
+ const normalized = options.allowDotPrefix ? trimmed.replaceAll("\\", "/").replace(/^\.\/+/, "") : trimmed.replaceAll("\\", "/");
1739
+ if (normalized.length === 0) return null;
1740
+ const parts = normalized.split("/");
1741
+ if (parts.some((part) => part.length === 0 || part === "." || part === "..")) return null;
1742
+ return parts.join("/");
1743
+ }
1744
+ function refreshStaticChunkCollections(result) {
1745
+ replaceStaticChunkCollection(
1746
+ result,
1747
+ "textChunks",
1748
+ result.chunks.filter((chunk) => chunk.type === "text")
1749
+ );
1750
+ replaceStaticChunkCollection(
1751
+ result,
1752
+ "imageChunks",
1753
+ result.chunks.filter((chunk) => chunk.type === "image")
1754
+ );
1755
+ replaceStaticChunkCollection(
1756
+ result,
1757
+ "tableChunks",
1758
+ result.chunks.filter((chunk) => chunk.type === "table")
1759
+ );
1760
+ replaceStaticChunkCollection(
1761
+ result,
1762
+ "pageChunks",
1763
+ result.chunks.filter((chunk) => chunk.type === "page")
1764
+ );
1765
+ }
1766
+ function replaceStaticChunkCollection(result, property, value) {
1767
+ const descriptor = Object.getOwnPropertyDescriptor(result, property);
1768
+ if (!descriptor || descriptor.get || descriptor.writable !== true) return;
1769
+ Object.defineProperty(result, property, {
1770
+ ...descriptor,
1771
+ value
1772
+ });
1773
+ }
1774
+ function getImageContentType(filePath) {
1775
+ const extension = filePath.split(".").at(-1)?.toLowerCase();
1776
+ if (extension === "jpg" || extension === "jpeg") return "image/jpeg";
1777
+ if (extension === "png") return "image/png";
1778
+ if (extension === "webp") return "image/webp";
1779
+ if (extension === "gif") return "image/gif";
1780
+ if (extension === "svg") return "image/svg+xml";
1781
+ return defaultImageContentType;
1782
+ }
1783
+ function isRecord(value) {
1784
+ return typeof value === "object" && value !== null;
1785
+ }
1786
+
1787
+ // src/storage/parsed-document-storage.ts
1788
+ import JSZip3 from "jszip";
1789
+ var parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
1790
+ var jsonContentType = "application/json; charset=utf-8";
1791
+ var markdownContentType = "text/markdown; charset=utf-8";
1792
+ var csvContentType = "text/csv; charset=utf-8";
1793
+ var htmlContentType = "text/html; charset=utf-8";
1794
+ var binaryContentType = "application/octet-stream";
1795
+ async function syncParseResultToParsedDocumentStorage(params) {
1796
+ const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
1797
+ const revisionKey = params.revisionKey ?? params.result.jobId;
1798
+ const objects = await collectResultLayoutObjects(params.result);
1799
+ for (const object of objects) {
1800
+ await params.storage.writeObject({
1801
+ documentId,
1802
+ revisionKey,
1803
+ ...object
1804
+ });
1805
+ }
1806
+ const commit = {
1807
+ version: 1,
1808
+ documentId,
1809
+ revisionKey,
1810
+ source: params.source ?? "resultZip",
1811
+ committedAt: (/* @__PURE__ */ new Date()).toISOString()
1812
+ };
1813
+ await writeParsedDocumentJsonObject(params.storage, {
1814
+ documentId,
1815
+ revisionKey,
1816
+ path: parsedDocumentCommitPath,
1817
+ value: commit
1818
+ });
1819
+ return {
1820
+ result: params.result,
1821
+ documentId,
1822
+ revisionKey,
1823
+ commit
1824
+ };
1825
+ }
1826
+ async function readParsedDocumentCommit(params) {
1827
+ const object = await params.storage.readObject({
1828
+ documentId: params.documentId,
1829
+ revisionKey: params.revisionKey,
1830
+ path: parsedDocumentCommitPath
1831
+ });
1832
+ if (!object) {
1833
+ return null;
1834
+ }
1835
+ const commit = readJsonObject(object);
1836
+ return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
1837
+ }
1838
+ function readJsonObject(object) {
1839
+ return JSON.parse(Buffer.from(object.body).toString("utf8"));
1840
+ }
1841
+ async function collectResultLayoutObjects(result) {
1842
+ const objectsByPath = /* @__PURE__ */ new Map();
1843
+ const rawZipObjects = await collectRawZipObjects(result.rawZip);
1844
+ for (const object of rawZipObjects) {
1845
+ objectsByPath.set(object.path, object);
1846
+ }
1847
+ for (const object of collectGeneratedResultObjects(result)) {
1848
+ if (!objectsByPath.has(object.path)) {
1849
+ objectsByPath.set(object.path, object);
1850
+ }
1851
+ }
1852
+ return [...objectsByPath.values()].filter((object) => !isSdkMetadataPath(object.path));
1853
+ }
1854
+ async function collectRawZipObjects(rawZip) {
1855
+ try {
1856
+ const zip = await JSZip3.loadAsync(rawZip);
1857
+ const objects = [];
1858
+ for (const entry of Object.values(zip.files)) {
1859
+ if (entry.dir || entry.name === "result.zip") {
1860
+ continue;
1861
+ }
1862
+ objects.push({
1863
+ path: normalizeResultPath(entry.name),
1864
+ body: new Uint8Array(await entry.async("nodebuffer")),
1865
+ contentType: inferContentTypeFromPath(entry.name)
1866
+ });
1867
+ }
1868
+ return objects;
1869
+ } catch {
1870
+ return [];
1871
+ }
1872
+ }
1873
+ function collectGeneratedResultObjects(result) {
1874
+ const objects = [
1875
+ createJsonObject("manifest.json", result.manifest),
1876
+ createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)))
1877
+ ];
1878
+ if (result.docNav) {
1879
+ objects.push(createJsonObject("doc_nav.json", result.docNav));
1880
+ }
1881
+ if (result.fullMarkdown) {
1882
+ objects.push(createTextObject("full.md", result.fullMarkdown, markdownContentType));
1883
+ }
1884
+ if (result.chunksSlim) {
1885
+ objects.push(createJsonObject("chunks_slim.json", { chunks: result.chunksSlim }));
1886
+ }
1887
+ if (result.hierarchy) {
1888
+ objects.push(createJsonObject("hierarchy.json", result.hierarchy));
1889
+ }
1890
+ if (result.tocHierarchies) {
1891
+ objects.push(createJsonObject("toc_hierarchies.json", result.tocHierarchies));
1892
+ }
1893
+ if (result.kbCsv) {
1894
+ objects.push(createTextObject("kb.csv", result.kbCsv, csvContentType));
1895
+ }
1896
+ if (result.hierarchyViewHtml) {
1897
+ objects.push(
1898
+ createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType)
1899
+ );
1900
+ }
1901
+ for (const chunk of result.imageChunks) {
1902
+ objects.push({
1903
+ path: normalizeResultPath(chunk.filePath),
1904
+ body: new Uint8Array(chunk.data),
1905
+ contentType: inferContentTypeFromPath(chunk.filePath),
1906
+ metadata: {
1907
+ chunkId: chunk.chunkId,
1908
+ chunkType: chunk.type,
1909
+ sourcePath: chunk.filePath
1910
+ }
1911
+ });
1912
+ }
1913
+ for (const chunk of result.tableChunks) {
1914
+ objects.push({
1915
+ path: normalizeResultPath(chunk.filePath),
1916
+ body: Buffer.from(chunk.html, "utf8"),
1917
+ contentType: htmlContentType,
1918
+ metadata: {
1919
+ chunkId: chunk.chunkId,
1920
+ chunkType: chunk.type,
1921
+ sourcePath: chunk.filePath
1922
+ }
1923
+ });
1924
+ }
1925
+ return objects;
1926
+ }
1927
+ function serializeChunks2(chunks) {
1928
+ return {
1929
+ chunks: chunks.map((chunk) => {
1930
+ const rawChunk = {
1931
+ chunkId: chunk.chunkId,
1932
+ type: chunk.type,
1933
+ contentSource: chunk.contentSource,
1934
+ content: chunk.content,
1935
+ path: chunk.path,
1936
+ metadata: chunk.metadata
1937
+ };
1938
+ if (chunk.type === "image" || chunk.type === "table") {
1939
+ return {
1940
+ ...rawChunk,
1941
+ filePath: chunk.filePath,
1942
+ assetUrl: chunk.assetUrl
1943
+ };
1944
+ }
1945
+ return rawChunk;
1946
+ })
1947
+ };
1948
+ }
1949
+ async function writeParsedDocumentJsonObject(storage, params) {
1950
+ await storage.writeObject({
1951
+ documentId: params.documentId,
1952
+ revisionKey: params.revisionKey,
1953
+ path: params.path,
1954
+ body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
1955
+ contentType: jsonContentType
1956
+ });
1957
+ }
1958
+ function createJsonObject(path4, value) {
1959
+ return {
1960
+ path: path4,
1961
+ body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
1962
+ contentType: jsonContentType
1963
+ };
1964
+ }
1965
+ function createTextObject(path4, value, contentType) {
1966
+ return {
1967
+ path: path4,
1968
+ body: Buffer.from(value, "utf8"),
1969
+ contentType
1970
+ };
1971
+ }
1972
+ function inferContentTypeFromPath(filePath) {
1973
+ const lower = filePath.toLowerCase();
1974
+ if (lower.endsWith(".json")) return jsonContentType;
1975
+ if (lower.endsWith(".md")) return markdownContentType;
1976
+ if (lower.endsWith(".csv")) return csvContentType;
1977
+ if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
1978
+ if (lower.endsWith(".png")) return "image/png";
1979
+ if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
1980
+ if (lower.endsWith(".gif")) return "image/gif";
1981
+ if (lower.endsWith(".webp")) return "image/webp";
1982
+ if (lower.endsWith(".svg")) return "image/svg+xml";
1983
+ return binaryContentType;
1984
+ }
1985
+ function normalizeResultPath(value) {
1986
+ const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
1987
+ if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
1988
+ throw new Error(`Invalid parsed result path: ${value}`);
1989
+ }
1990
+ return normalized;
1991
+ }
1992
+ function isSdkMetadataPath(path4) {
1993
+ return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
1994
+ }
1995
+
1509
1996
  // src/knowledge/local-store.ts
1510
1997
  import { createHash } from "crypto";
1511
1998
  import os from "os";
@@ -1731,27 +2218,61 @@ function isMissingFileError2(error) {
1731
2218
  // src/knowledge/knowledge.ts
1732
2219
  var DEFAULT_READ_LIMIT = 12;
1733
2220
  var MAX_READ_LIMIT = 40;
2221
+ var DEFAULT_PAGE_SIZE = 20;
2222
+ var MAX_PAGE_SIZE = 200;
1734
2223
  var DEFAULT_GREP_LIMIT = 20;
1735
2224
  var MAX_GREP_LIMIT = 50;
1736
2225
  var DEFAULT_CONTEXT_CHARS = 80;
2226
+ var DEFAULT_REMOTE_SCAN_PAGE_SIZE = 100;
2227
+ var DEFAULT_MAX_SYNC_PAGES = 10;
2228
+ var DEFAULT_SYNC_DEADLINE_MS = 8e3;
2229
+ var DEFAULT_GREP_MAX_PAGES = 50;
2230
+ var DEFAULT_GREP_DEADLINE_MS = 8e3;
2231
+ var DEFAULT_OUTLINE_MAX_PAGES = 50;
2232
+ var DEFAULT_OUTLINE_DEADLINE_MS = 8e3;
1737
2233
  var SAFE_LOCAL_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
1738
2234
  var REMOTE_DOCUMENT_ID_PATTERN = /^doc[_-]/;
2235
+ var remoteReconstructionPageDirectory = ".knowhere-sdk/remote-reconstruction/pages";
1739
2236
  var Knowledge = class _Knowledge {
1740
2237
  client;
1741
2238
  store;
2239
+ cacheDirectory;
2240
+ parsedStorageConfig;
1742
2241
  constructor(client, options) {
1743
2242
  this.client = client;
2243
+ this.cacheDirectory = options?.cacheDirectory;
2244
+ this.parsedStorageConfig = options?.parsedStorage;
1744
2245
  this.store = new LocalKnowledgeStore(options?.cacheDirectory);
1745
2246
  }
1746
2247
  withCacheDirectory(cacheDirectory) {
1747
- return new _Knowledge(this.client, { cacheDirectory });
2248
+ return new _Knowledge(this.client, {
2249
+ cacheDirectory,
2250
+ parsedStorage: this.parsedStorageConfig
2251
+ });
1748
2252
  }
1749
- async parse(params) {
1750
- const result = await this.client.parse(params);
2253
+ withParsedStorage(parsedStorage) {
2254
+ return new _Knowledge(this.client, {
2255
+ cacheDirectory: this.cacheDirectory,
2256
+ parsedStorage
2257
+ });
2258
+ }
2259
+ async parseToLocalCache(params) {
2260
+ const loadedResult = params.storageAdapter ? await this.parseWithoutClientStorageAdapter(params) : await this.client.parse(params);
2261
+ const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter);
2262
+ const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result);
2263
+ const result = syncedStorage.result;
1751
2264
  const document = await this.store.saveResult(result, {
1752
2265
  localDocumentId: params.localDocumentId
1753
2266
  });
1754
- return { document, result };
2267
+ return {
2268
+ document,
2269
+ result,
2270
+ assetUrlsByFilePath: storedAssets.assetUrlsByFilePath
2271
+ };
2272
+ }
2273
+ /** @deprecated Use parseToLocalCache for local cache imports or the top-level client.parse for server-safe parsing. */
2274
+ async parse(params) {
2275
+ return this.parseToLocalCache(params);
1755
2276
  }
1756
2277
  async startParse(params) {
1757
2278
  const job = await this.client.startParse(params);
@@ -1782,14 +2303,95 @@ var Knowledge = class _Knowledge {
1782
2303
  results
1783
2304
  };
1784
2305
  }
1785
- async cacheJobResult(params) {
1786
- const result = await this.client.jobs.load(params.jobId, {
2306
+ async loadJobResult(params) {
2307
+ const loadedResult = await this.client.jobs.load(params.jobId, {
1787
2308
  verifyChecksum: params.verifyChecksum
1788
2309
  });
2310
+ const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter);
2311
+ const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result);
2312
+ const result = syncedStorage.result;
2313
+ return {
2314
+ result,
2315
+ assetUrlsByFilePath: storedAssets.assetUrlsByFilePath
2316
+ };
2317
+ }
2318
+ async importJobResult(params) {
2319
+ const stored = await this.loadJobResult(params);
2320
+ const result = stored.result;
1789
2321
  const document = await this.store.saveResult(result, {
1790
2322
  localDocumentId: params.localDocumentId
1791
2323
  });
1792
- return { document, result };
2324
+ return {
2325
+ document,
2326
+ result,
2327
+ assetUrlsByFilePath: stored.assetUrlsByFilePath
2328
+ };
2329
+ }
2330
+ /** @deprecated Use importJobResult for local cache imports or loadJobResult for server-safe loads. */
2331
+ async cacheJobResult(params) {
2332
+ return this.importJobResult(params);
2333
+ }
2334
+ async syncParsedDocument(params) {
2335
+ const normalized = normalizeDocumentReference(params);
2336
+ const parsedStorageConfig = this.requireParsedStorageConfig();
2337
+ if (normalized.documentId) {
2338
+ return this.syncRemoteDocumentToParsedStorage({
2339
+ documentId: normalized.documentId,
2340
+ revisionKey: params.revisionKey,
2341
+ parsedStorageConfig
2342
+ });
2343
+ }
2344
+ if (normalized.jobId) {
2345
+ const loadedResult = await this.client.jobs.load(normalized.jobId);
2346
+ const synced2 = await syncParseResultToParsedDocumentStorage({
2347
+ result: loadedResult,
2348
+ storage: parsedStorageConfig.storage,
2349
+ documentId: loadedResult.documentId ?? normalized.localDocumentId ?? normalized.jobId,
2350
+ revisionKey: params.revisionKey ?? normalized.jobId
2351
+ });
2352
+ return {
2353
+ documentId: synced2.documentId,
2354
+ revisionKey: synced2.revisionKey,
2355
+ completed: true,
2356
+ commit: synced2.commit
2357
+ };
2358
+ }
2359
+ if (!normalized.localDocumentId) {
2360
+ throw new ValidationError("localDocumentId, documentId, or jobId is required");
2361
+ }
2362
+ const { result } = await this.store.loadResult(normalized.localDocumentId);
2363
+ const synced = await syncParseResultToParsedDocumentStorage({
2364
+ result,
2365
+ storage: parsedStorageConfig.storage,
2366
+ documentId: result.documentId ?? normalized.localDocumentId,
2367
+ revisionKey: params.revisionKey ?? result.jobId
2368
+ });
2369
+ return {
2370
+ documentId: synced.documentId,
2371
+ revisionKey: synced.revisionKey,
2372
+ completed: true,
2373
+ commit: synced.commit
2374
+ };
2375
+ }
2376
+ async syncParseResultToConfiguredStorage(result) {
2377
+ const parsedStorageConfig = this.parsedStorageConfig;
2378
+ if (!parsedStorageConfig) {
2379
+ return { result };
2380
+ }
2381
+ const synced = await syncParseResultToParsedDocumentStorage({
2382
+ result,
2383
+ storage: parsedStorageConfig.storage
2384
+ });
2385
+ return {
2386
+ result: synced.result,
2387
+ commit: synced.commit
2388
+ };
2389
+ }
2390
+ async parseWithoutClientStorageAdapter(params) {
2391
+ const parseParams = { ...params };
2392
+ delete parseParams.localDocumentId;
2393
+ delete parseParams.storageAdapter;
2394
+ return this.client.parse(parseParams);
1793
2395
  }
1794
2396
  async cacheDocument(params) {
1795
2397
  if (!params.documentId) {
@@ -1799,7 +2401,7 @@ var Knowledge = class _Knowledge {
1799
2401
  params.documentId
1800
2402
  );
1801
2403
  const jobId = await this.getPublishedDocumentJobId(params.documentId);
1802
- return this.cacheJobResult({
2404
+ return this.importJobResult({
1803
2405
  jobId,
1804
2406
  localDocumentId: params.localDocumentId ?? existing?.localDocumentId ?? createLocalDocumentIdForRemote(params.documentId)
1805
2407
  });
@@ -1836,7 +2438,7 @@ var Knowledge = class _Knowledge {
1836
2438
  };
1837
2439
  }
1838
2440
  try {
1839
- const cached = await this.cacheJobResult({
2441
+ const cached = await this.importJobResult({
1840
2442
  jobId,
1841
2443
  localDocumentId: trackedJob.localDocumentId
1842
2444
  });
@@ -1861,6 +2463,13 @@ var Knowledge = class _Knowledge {
1861
2463
  return this.store.listDocuments();
1862
2464
  }
1863
2465
  async getDocumentOutline(reference) {
2466
+ const normalized = normalizeDocumentReference(reference);
2467
+ if (normalized.documentId) {
2468
+ return this.getRemoteDocumentOutline(normalized.documentId, normalized.revisionKey);
2469
+ }
2470
+ if (normalized.localDocumentId && looksLikeRemoteDocumentId(normalized.localDocumentId) && !await this.store.getDocument(normalized.localDocumentId)) {
2471
+ return this.getRemoteDocumentOutline(normalized.localDocumentId, normalized.revisionKey);
2472
+ }
1864
2473
  const { document, result } = await this.loadReadableResult(reference);
1865
2474
  const chunks = indexChunks(result);
1866
2475
  const sections = buildFlatSections(result, chunks);
@@ -1876,6 +2485,14 @@ var Knowledge = class _Knowledge {
1876
2485
  };
1877
2486
  }
1878
2487
  async readChunks(params) {
2488
+ validateReadParams(params);
2489
+ const normalized = normalizeDocumentReference(params);
2490
+ if (normalized.documentId) {
2491
+ return this.readRemoteDocumentChunks(normalized.documentId, params);
2492
+ }
2493
+ if (normalized.localDocumentId && looksLikeRemoteDocumentId(normalized.localDocumentId) && !await this.store.getDocument(normalized.localDocumentId)) {
2494
+ return this.readRemoteDocumentChunks(normalized.localDocumentId, params);
2495
+ }
1879
2496
  const { document, result } = await this.loadReadableResult(params);
1880
2497
  const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT);
1881
2498
  const chunks = indexChunks(result).filter((chunk) => matchesReadScope(chunk, params));
@@ -1892,6 +2509,13 @@ var Knowledge = class _Knowledge {
1892
2509
  if (!params.pattern) {
1893
2510
  throw new ValidationError("pattern is required");
1894
2511
  }
2512
+ const normalized = normalizeDocumentReference(params);
2513
+ if (normalized.documentId) {
2514
+ return this.grepRemoteDocumentChunks(normalized.documentId, params);
2515
+ }
2516
+ if (normalized.localDocumentId && looksLikeRemoteDocumentId(normalized.localDocumentId) && !await this.store.getDocument(normalized.localDocumentId)) {
2517
+ return this.grepRemoteDocumentChunks(normalized.localDocumentId, params);
2518
+ }
1895
2519
  const { document, result } = await this.loadReadableResult(params);
1896
2520
  const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT);
1897
2521
  const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS;
@@ -1971,7 +2595,7 @@ var Knowledge = class _Knowledge {
1971
2595
  });
1972
2596
  }
1973
2597
  if (normalized.jobId) {
1974
- return this.cacheJobResult({
2598
+ return this.importJobResult({
1975
2599
  jobId: normalized.jobId,
1976
2600
  localDocumentId: normalized.localDocumentId
1977
2601
  });
@@ -1991,21 +2615,657 @@ var Knowledge = class _Knowledge {
1991
2615
  }
1992
2616
  throw new Error(`Local Knowhere document not found: ${normalized.localDocumentId}`);
1993
2617
  }
1994
- async findLocalDocumentByRemoteId(documentId) {
1995
- const documents = await this.store.listDocuments();
1996
- return documents.find((document) => document.documentId === documentId);
2618
+ async readRemoteDocumentChunks(documentId, params) {
2619
+ const explicitRevisionKey = params.revisionKey ?? params.jobId;
2620
+ if (explicitRevisionKey) {
2621
+ const stored2 = await this.tryReadChunksFromParsedStorage({
2622
+ documentId,
2623
+ revisionKey: explicitRevisionKey,
2624
+ params
2625
+ });
2626
+ if (stored2) {
2627
+ return stored2;
2628
+ }
2629
+ }
2630
+ if (hasScanReadFilters(params)) {
2631
+ return this.scanRemoteDocumentChunks(documentId, params);
2632
+ }
2633
+ const pageParams = getReadPageParams(params);
2634
+ const remotePage = await this.client.documents.listChunks(documentId, {
2635
+ page: pageParams.page,
2636
+ pageSize: pageParams.pageSize,
2637
+ chunkType: params.chunkType,
2638
+ includeAssetUrls: true
2639
+ });
2640
+ const revisionKey = getRemoteRevisionKey(remotePage);
2641
+ const stored = await this.tryReadChunksFromParsedStorage({
2642
+ documentId,
2643
+ revisionKey,
2644
+ params
2645
+ });
2646
+ if (stored) {
2647
+ return stored;
2648
+ }
2649
+ const chunks = remotePage.chunks.map((chunk) => toRemoteReadChunk(chunk));
2650
+ this.scheduleParsedStorageSync(documentId, revisionKey);
2651
+ return {
2652
+ document: createRemoteKnowledgeDocument({
2653
+ documentId,
2654
+ response: remotePage,
2655
+ revisionKey
2656
+ }),
2657
+ chunks,
2658
+ page: remotePage.pagination.page,
2659
+ pageSize: remotePage.pagination.pageSize,
2660
+ totalChunks: remotePage.pagination.total,
2661
+ totalPages: remotePage.pagination.totalPages
2662
+ };
1997
2663
  }
1998
- async getPublishedDocumentJobId(documentId) {
1999
- const response = await this.client.documents.listChunks(documentId, {
2664
+ async scanRemoteDocumentChunks(documentId, params) {
2665
+ const explicitRevisionKey = params.revisionKey ?? params.jobId;
2666
+ if (explicitRevisionKey) {
2667
+ const stored2 = await this.tryReadChunksFromParsedStorage({
2668
+ documentId,
2669
+ revisionKey: explicitRevisionKey,
2670
+ params
2671
+ });
2672
+ if (stored2) {
2673
+ return stored2;
2674
+ }
2675
+ }
2676
+ const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT);
2677
+ const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
2678
+ const firstPage = await this.client.documents.listChunks(documentId, {
2000
2679
  page: 1,
2001
- pageSize: 1
2680
+ pageSize: limits.remotePageSize,
2681
+ chunkType: params.chunkType,
2682
+ includeAssetUrls: true
2002
2683
  });
2003
- if (!response.jobId) {
2004
- throw new Error(
2005
- `Cannot sync server document ${documentId}: current published job id was not returned.`
2006
- );
2684
+ const revisionKey = getRemoteRevisionKey(firstPage);
2685
+ const stored = await this.tryReadChunksFromParsedStorage({
2686
+ documentId,
2687
+ revisionKey,
2688
+ params
2689
+ });
2690
+ if (stored) {
2691
+ return stored;
2007
2692
  }
2008
- return response.jobId;
2693
+ const selected = [];
2694
+ let scannedPages = 0;
2695
+ let page = 1;
2696
+ let response = firstPage;
2697
+ const startedAt = Date.now();
2698
+ while (true) {
2699
+ scannedPages += 1;
2700
+ selected.push(...response.chunks.filter((chunk) => matchesRemoteReadScope(chunk, params)));
2701
+ if (selected.length >= limit || response.pagination.page >= response.pagination.totalPages || scannedPages >= limits.outlineMaxPages || Date.now() - startedAt >= limits.outlineDeadlineMs) {
2702
+ break;
2703
+ }
2704
+ page += 1;
2705
+ response = await this.client.documents.listChunks(documentId, {
2706
+ page,
2707
+ pageSize: limits.remotePageSize,
2708
+ chunkType: params.chunkType,
2709
+ includeAssetUrls: true
2710
+ });
2711
+ }
2712
+ const chunks = selected.slice(0, limit).map((chunk) => toRemoteReadChunk(chunk));
2713
+ this.scheduleParsedStorageSync(documentId, revisionKey);
2714
+ return {
2715
+ document: createRemoteKnowledgeDocument({
2716
+ documentId,
2717
+ response: firstPage,
2718
+ revisionKey
2719
+ }),
2720
+ chunks,
2721
+ nextChunk: selected.length > limit ? toChunkPosition(selected[limit]) : void 0,
2722
+ totalChunks: firstPage.pagination.total,
2723
+ totalPages: firstPage.pagination.totalPages
2724
+ };
2725
+ }
2726
+ async grepRemoteDocumentChunks(documentId, params) {
2727
+ const explicitRevisionKey = params.revisionKey ?? params.jobId;
2728
+ if (explicitRevisionKey && !params.continuationCursor) {
2729
+ const stored2 = await this.tryGrepChunksFromParsedStorage({
2730
+ documentId,
2731
+ revisionKey: explicitRevisionKey,
2732
+ params
2733
+ });
2734
+ if (stored2) {
2735
+ return stored2;
2736
+ }
2737
+ }
2738
+ const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT);
2739
+ const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS;
2740
+ const matcher = createMatcher(params);
2741
+ const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
2742
+ const cursor = parseContinuationCursor(params.continuationCursor);
2743
+ const startPage = cursor?.documentId === documentId ? cursor.nextPage : 1;
2744
+ const firstPage = await this.client.documents.listChunks(documentId, {
2745
+ page: startPage,
2746
+ pageSize: limits.remotePageSize,
2747
+ chunkType: params.chunkType,
2748
+ includeAssetUrls: true
2749
+ });
2750
+ const revisionKey = getRemoteRevisionKey(firstPage);
2751
+ const stored = startPage === 1 ? await this.tryGrepChunksFromParsedStorage({
2752
+ documentId,
2753
+ revisionKey,
2754
+ params
2755
+ }) : null;
2756
+ if (stored) {
2757
+ return stored;
2758
+ }
2759
+ const matches = [];
2760
+ let scannedChunks = 0;
2761
+ let scannedPages = 0;
2762
+ let response = firstPage;
2763
+ let page = startPage;
2764
+ const startedAt = Date.now();
2765
+ while (true) {
2766
+ scannedPages += 1;
2767
+ for (const [chunkIndex, chunk] of response.chunks.entries()) {
2768
+ if (response.pagination.page === startPage && cursor?.nextChunkIndex !== void 0 && chunkIndex < cursor.nextChunkIndex) {
2769
+ continue;
2770
+ }
2771
+ if (!matchesGrepScope(toIndexedRemoteChunk(chunk), params)) {
2772
+ continue;
2773
+ }
2774
+ scannedChunks += 1;
2775
+ const chunkMatches = matcher(chunk.content ?? "");
2776
+ for (const [matchIndex, match] of chunkMatches.entries()) {
2777
+ if (response.pagination.page === startPage && cursor?.nextChunkIndex === chunkIndex && cursor.nextMatchIndex !== void 0 && matchIndex < cursor.nextMatchIndex) {
2778
+ continue;
2779
+ }
2780
+ matches.push({
2781
+ position: toChunkPosition(chunk),
2782
+ chunkId: chunk.chunkId,
2783
+ chunkType: chunk.chunkType,
2784
+ sectionPath: chunk.sectionPath ?? "",
2785
+ sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? "",
2786
+ filePath: chunk.filePath ?? void 0,
2787
+ startOffset: match.startOffset,
2788
+ endOffset: match.endOffset,
2789
+ snippet: buildSnippet(
2790
+ chunk.content ?? "",
2791
+ match.startOffset,
2792
+ match.endOffset,
2793
+ contextChars
2794
+ )
2795
+ });
2796
+ if (matches.length >= maxResults) {
2797
+ const continuationCursor = createGrepContinuationCursor({
2798
+ documentId,
2799
+ revisionKey,
2800
+ response,
2801
+ chunkIndex,
2802
+ matchIndex,
2803
+ chunkMatchCount: chunkMatches.length
2804
+ });
2805
+ this.scheduleParsedStorageSync(documentId, revisionKey);
2806
+ return {
2807
+ document: createRemoteKnowledgeDocument({
2808
+ documentId,
2809
+ response: firstPage,
2810
+ revisionKey
2811
+ }),
2812
+ matches,
2813
+ scannedChunks,
2814
+ truncated: continuationCursor !== void 0,
2815
+ continuationCursor
2816
+ };
2817
+ }
2818
+ }
2819
+ }
2820
+ const shouldStop = response.pagination.page >= response.pagination.totalPages || scannedPages >= limits.grepMaxPages || Date.now() - startedAt >= limits.grepDeadlineMs;
2821
+ if (shouldStop) {
2822
+ const truncated = response.pagination.page < response.pagination.totalPages;
2823
+ this.scheduleParsedStorageSync(documentId, revisionKey);
2824
+ return {
2825
+ document: createRemoteKnowledgeDocument({
2826
+ documentId,
2827
+ response: firstPage,
2828
+ revisionKey
2829
+ }),
2830
+ matches,
2831
+ scannedChunks,
2832
+ truncated,
2833
+ continuationCursor: truncated ? createContinuationCursor({
2834
+ documentId,
2835
+ revisionKey,
2836
+ nextPage: response.pagination.page + 1
2837
+ }) : void 0
2838
+ };
2839
+ }
2840
+ page += 1;
2841
+ response = await this.client.documents.listChunks(documentId, {
2842
+ page,
2843
+ pageSize: limits.remotePageSize,
2844
+ chunkType: params.chunkType,
2845
+ includeAssetUrls: true
2846
+ });
2847
+ }
2848
+ }
2849
+ async getRemoteDocumentOutline(documentId, revisionKeyOverride) {
2850
+ if (revisionKeyOverride) {
2851
+ const stored2 = await this.tryReadOutlineFromParsedStorage(documentId, revisionKeyOverride);
2852
+ if (stored2) {
2853
+ return stored2;
2854
+ }
2855
+ }
2856
+ const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
2857
+ const firstPage = await this.client.documents.listChunks(documentId, {
2858
+ page: 1,
2859
+ pageSize: limits.remotePageSize,
2860
+ includeAssetUrls: true
2861
+ });
2862
+ const revisionKey = getRemoteRevisionKey(firstPage);
2863
+ const stored = await this.tryReadOutlineFromParsedStorage(documentId, revisionKey);
2864
+ if (stored) {
2865
+ return stored;
2866
+ }
2867
+ const chunks = [];
2868
+ let page = 1;
2869
+ let response = firstPage;
2870
+ let scannedPages = 0;
2871
+ const startedAt = Date.now();
2872
+ while (true) {
2873
+ scannedPages += 1;
2874
+ chunks.push(...response.chunks.map(toIndexedRemoteChunk));
2875
+ if (response.pagination.page >= response.pagination.totalPages || scannedPages >= limits.outlineMaxPages || Date.now() - startedAt >= limits.outlineDeadlineMs) {
2876
+ break;
2877
+ }
2878
+ page += 1;
2879
+ response = await this.client.documents.listChunks(documentId, {
2880
+ page,
2881
+ pageSize: limits.remotePageSize,
2882
+ includeAssetUrls: true
2883
+ });
2884
+ }
2885
+ const truncated = response.pagination.page < response.pagination.totalPages;
2886
+ const sections = buildFlatSectionsFromIndexedChunks(chunks);
2887
+ this.scheduleParsedStorageSync(documentId, revisionKey);
2888
+ return {
2889
+ document: createRemoteKnowledgeDocument({
2890
+ documentId,
2891
+ response: firstPage,
2892
+ revisionKey,
2893
+ typeCounts: countIndexedTypes(chunks)
2894
+ }),
2895
+ totalChunks: firstPage.pagination.total,
2896
+ typeCounts: countIndexedTypes(chunks),
2897
+ sections,
2898
+ sectionTree: nestSections(sections),
2899
+ truncated,
2900
+ continuationCursor: truncated ? createContinuationCursor({
2901
+ documentId,
2902
+ revisionKey,
2903
+ nextPage: response.pagination.page + 1
2904
+ }) : void 0
2905
+ };
2906
+ }
2907
+ async tryReadChunksFromParsedStorage(params) {
2908
+ const storedResult = await this.readCommittedParsedStorageResult(
2909
+ params.documentId,
2910
+ params.revisionKey
2911
+ );
2912
+ if (!storedResult) return null;
2913
+ const pageParams = getReadPageParams(params.params);
2914
+ const scopedChunks = storedResult.chunks.filter((chunk) => !params.params.chunkType || chunk.chunkType === params.params.chunkType).filter((chunk) => matchesReadScope(chunk, params.params));
2915
+ const selected = hasScanReadFilters(params.params) ? selectReadWindow(
2916
+ scopedChunks,
2917
+ params.params,
2918
+ clampLimit(params.params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT)
2919
+ ) : scopedChunks.slice(
2920
+ (pageParams.page - 1) * pageParams.pageSize,
2921
+ pageParams.page * pageParams.pageSize
2922
+ );
2923
+ const visibleChunks = await applyStoredAssetUrlsToReadChunks({
2924
+ storageConfig: this.requireParsedStorageConfig(),
2925
+ documentId: params.documentId,
2926
+ revisionKey: params.revisionKey,
2927
+ chunks: selected.map(toReadChunk)
2928
+ });
2929
+ const totalChunks = scopedChunks.length;
2930
+ return {
2931
+ document: createStoredKnowledgeDocument({
2932
+ documentId: params.documentId,
2933
+ storedResult,
2934
+ revisionKey: params.revisionKey
2935
+ }),
2936
+ chunks: visibleChunks,
2937
+ page: hasScanReadFilters(params.params) ? void 0 : pageParams.page,
2938
+ pageSize: hasScanReadFilters(params.params) ? void 0 : pageParams.pageSize,
2939
+ totalChunks,
2940
+ totalPages: Math.max(1, Math.ceil(totalChunks / pageParams.pageSize)),
2941
+ nextChunk: getNextChunkPosition(scopedChunks, selected)
2942
+ };
2943
+ }
2944
+ async tryGrepChunksFromParsedStorage(params) {
2945
+ const storedResult = await this.readCommittedParsedStorageResult(
2946
+ params.documentId,
2947
+ params.revisionKey
2948
+ );
2949
+ if (!storedResult) return null;
2950
+ const maxResults = clampLimit(params.params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT);
2951
+ const contextChars = params.params.contextChars ?? DEFAULT_CONTEXT_CHARS;
2952
+ const matcher = createMatcher(params.params);
2953
+ const scopedChunks = storedResult.chunks.filter((chunk) => !params.params.chunkType || chunk.chunkType === params.params.chunkType).filter((chunk) => matchesGrepScope(chunk, params.params));
2954
+ const matches = [];
2955
+ let scannedChunks = 0;
2956
+ for (const chunk of scopedChunks) {
2957
+ scannedChunks += 1;
2958
+ for (const match of matcher(chunk.content)) {
2959
+ matches.push({
2960
+ position: chunk.position,
2961
+ chunkId: chunk.chunkId,
2962
+ chunkType: chunk.chunkType,
2963
+ sectionPath: chunk.sectionPath,
2964
+ sourceChunkPath: chunk.sourceChunkPath,
2965
+ filePath: chunk.filePath,
2966
+ startOffset: match.startOffset,
2967
+ endOffset: match.endOffset,
2968
+ snippet: buildSnippet(chunk.content, match.startOffset, match.endOffset, contextChars)
2969
+ });
2970
+ if (matches.length >= maxResults) {
2971
+ return {
2972
+ document: createStoredKnowledgeDocument({
2973
+ documentId: params.documentId,
2974
+ storedResult,
2975
+ revisionKey: params.revisionKey
2976
+ }),
2977
+ matches,
2978
+ scannedChunks,
2979
+ truncated: true
2980
+ };
2981
+ }
2982
+ }
2983
+ }
2984
+ return {
2985
+ document: createStoredKnowledgeDocument({
2986
+ documentId: params.documentId,
2987
+ storedResult,
2988
+ revisionKey: params.revisionKey
2989
+ }),
2990
+ matches,
2991
+ scannedChunks,
2992
+ truncated: false
2993
+ };
2994
+ }
2995
+ async tryReadOutlineFromParsedStorage(documentId, revisionKey) {
2996
+ const storedResult = await this.readCommittedParsedStorageResult(documentId, revisionKey);
2997
+ if (!storedResult) return null;
2998
+ const sections = storedResult.docNav?.sections && storedResult.docNav.sections.length > 0 ? flattenSections(
2999
+ storedResult.docNav.sections.map(
3000
+ (section) => toKnowledgeSection(
3001
+ section,
3002
+ storedResult.chunks,
3003
+ storedResult.manifest.sourceFileName
3004
+ )
3005
+ )
3006
+ ) : buildFlatSectionsFromIndexedChunks(storedResult.chunks);
3007
+ return {
3008
+ document: createStoredKnowledgeDocument({
3009
+ documentId,
3010
+ storedResult,
3011
+ revisionKey
3012
+ }),
3013
+ totalChunks: storedResult.chunks.length,
3014
+ typeCounts: storedResult.typeCounts,
3015
+ sections,
3016
+ sectionTree: storedResult.docNav?.sections && storedResult.docNav.sections.length > 0 ? storedResult.docNav.sections.map(
3017
+ (section) => toKnowledgeSection(
3018
+ section,
3019
+ storedResult.chunks,
3020
+ storedResult.manifest.sourceFileName
3021
+ )
3022
+ ) : nestSections(sections),
3023
+ truncated: false
3024
+ };
3025
+ }
3026
+ async readCommittedParsedStorageResult(documentId, revisionKey) {
3027
+ const parsedStorageConfig = this.parsedStorageConfig;
3028
+ if (!parsedStorageConfig) {
3029
+ return null;
3030
+ }
3031
+ const commit = await readParsedDocumentCommit({
3032
+ storage: parsedStorageConfig.storage,
3033
+ documentId,
3034
+ revisionKey
3035
+ });
3036
+ if (!commit) {
3037
+ return null;
3038
+ }
3039
+ const manifestObject = await parsedStorageConfig.storage.readObject({
3040
+ documentId,
3041
+ revisionKey,
3042
+ path: "manifest.json"
3043
+ });
3044
+ const chunksObject = await parsedStorageConfig.storage.readObject({
3045
+ documentId,
3046
+ revisionKey,
3047
+ path: "chunks.json"
3048
+ });
3049
+ if (!manifestObject || !chunksObject) {
3050
+ return null;
3051
+ }
3052
+ const manifest = readJsonObject(manifestObject);
3053
+ const chunks = parseStoredResultChunks({
3054
+ payload: readJsonObject(chunksObject),
3055
+ sourceFileName: manifest.sourceFileName
3056
+ });
3057
+ const docNavObject = await parsedStorageConfig.storage.readObject({
3058
+ documentId,
3059
+ revisionKey,
3060
+ path: "doc_nav.json"
3061
+ });
3062
+ const docNav = docNavObject ? readJsonObject(docNavObject) : void 0;
3063
+ return {
3064
+ commit,
3065
+ manifest,
3066
+ chunks,
3067
+ docNav: docNav?.sections ? { sections: docNav.sections } : void 0,
3068
+ typeCounts: countIndexedTypes(chunks)
3069
+ };
3070
+ }
3071
+ scheduleParsedStorageSync(documentId, revisionKey) {
3072
+ const parsedStorageConfig = this.parsedStorageConfig;
3073
+ if (!parsedStorageConfig) {
3074
+ return;
3075
+ }
3076
+ const scheduler = parsedStorageConfig.scheduler ?? defaultParsedStorageScheduler;
3077
+ const task = async () => {
3078
+ try {
3079
+ await this.syncRemoteDocumentToParsedStorage({
3080
+ documentId,
3081
+ revisionKey,
3082
+ parsedStorageConfig
3083
+ });
3084
+ } catch (error) {
3085
+ await parsedStorageConfig.storage.writeSyncProgress({
3086
+ documentId,
3087
+ revisionKey,
3088
+ nextChunkPage: 1,
3089
+ status: "failed",
3090
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
3091
+ error: error instanceof Error ? error.message : String(error)
3092
+ });
3093
+ }
3094
+ };
3095
+ void Promise.resolve(scheduler.schedule(task)).catch(() => void 0);
3096
+ }
3097
+ async syncRemoteDocumentToParsedStorage(params) {
3098
+ const firstPage = await this.client.documents.listChunks(params.documentId, {
3099
+ page: 1,
3100
+ pageSize: 1,
3101
+ includeAssetUrls: true
3102
+ });
3103
+ const revisionKey = params.revisionKey ?? getRemoteRevisionKey(firstPage);
3104
+ if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey) {
3105
+ return this.syncRemoteDocumentToParsedStorage({
3106
+ documentId: params.documentId,
3107
+ parsedStorageConfig: params.parsedStorageConfig
3108
+ });
3109
+ }
3110
+ if (firstPage.jobId) {
3111
+ try {
3112
+ const loadedResult = await this.client.jobs.load(firstPage.jobId);
3113
+ const synced = await syncParseResultToParsedDocumentStorage({
3114
+ result: loadedResult,
3115
+ storage: params.parsedStorageConfig.storage,
3116
+ documentId: params.documentId,
3117
+ revisionKey,
3118
+ source: "resultZip"
3119
+ });
3120
+ await params.parsedStorageConfig.storage.writeSyncProgress({
3121
+ documentId: params.documentId,
3122
+ revisionKey,
3123
+ status: "completed",
3124
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
3125
+ });
3126
+ return {
3127
+ documentId: synced.documentId,
3128
+ revisionKey: synced.revisionKey,
3129
+ completed: true,
3130
+ commit: synced.commit
3131
+ };
3132
+ } catch {
3133
+ }
3134
+ }
3135
+ return this.syncRemoteDocumentChunksToParsedStorage({
3136
+ documentId: params.documentId,
3137
+ revisionKey,
3138
+ firstPage,
3139
+ parsedStorageConfig: params.parsedStorageConfig
3140
+ });
3141
+ }
3142
+ async syncRemoteDocumentChunksToParsedStorage(params) {
3143
+ const limits = getParsedStorageLimits(params.parsedStorageConfig.limits);
3144
+ const existingProgress = await params.parsedStorageConfig.storage.readSyncProgress({
3145
+ documentId: params.documentId,
3146
+ revisionKey: params.revisionKey
3147
+ });
3148
+ let nextPage = existingProgress?.status === "running" ? existingProgress.nextChunkPage ?? 1 : 1;
3149
+ let response = nextPage === 1 && params.firstPage.pagination.pageSize === limits.remotePageSize ? params.firstPage : await this.client.documents.listChunks(params.documentId, {
3150
+ page: nextPage,
3151
+ pageSize: limits.remotePageSize,
3152
+ includeAssetUrls: true
3153
+ });
3154
+ const totalPages = response.pagination.totalPages;
3155
+ let syncedPages = 0;
3156
+ const startedAt = Date.now();
3157
+ while (nextPage <= totalPages) {
3158
+ if (syncedPages > 0) {
3159
+ response = await this.client.documents.listChunks(params.documentId, {
3160
+ page: nextPage,
3161
+ pageSize: limits.remotePageSize,
3162
+ includeAssetUrls: true
3163
+ });
3164
+ }
3165
+ await params.parsedStorageConfig.storage.writeObject({
3166
+ documentId: params.documentId,
3167
+ revisionKey: params.revisionKey,
3168
+ path: getRemoteReconstructionPagePath(response.pagination.page),
3169
+ body: Buffer.from(JSON.stringify({ chunks: response.chunks }), "utf8"),
3170
+ contentType: "application/json; charset=utf-8"
3171
+ });
3172
+ nextPage += 1;
3173
+ syncedPages += 1;
3174
+ await params.parsedStorageConfig.storage.writeSyncProgress({
3175
+ documentId: params.documentId,
3176
+ revisionKey: params.revisionKey,
3177
+ nextChunkPage: nextPage,
3178
+ status: "running",
3179
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
3180
+ });
3181
+ if (syncedPages >= limits.maxPagesPerSync || Date.now() - startedAt >= limits.syncDeadlineMs) {
3182
+ return {
3183
+ documentId: params.documentId,
3184
+ revisionKey: params.revisionKey,
3185
+ completed: false
3186
+ };
3187
+ }
3188
+ }
3189
+ const chunks = await this.readRemoteReconstructionChunks({
3190
+ documentId: params.documentId,
3191
+ revisionKey: params.revisionKey,
3192
+ totalPages,
3193
+ parsedStorageConfig: params.parsedStorageConfig
3194
+ });
3195
+ if (!chunks) {
3196
+ return {
3197
+ documentId: params.documentId,
3198
+ revisionKey: params.revisionKey,
3199
+ completed: false
3200
+ };
3201
+ }
3202
+ const synced = await syncParseResultToParsedDocumentStorage({
3203
+ result: createMinimalParseResultFromRemoteChunks({
3204
+ documentId: params.documentId,
3205
+ revisionKey: params.revisionKey,
3206
+ response,
3207
+ chunks
3208
+ }),
3209
+ storage: params.parsedStorageConfig.storage,
3210
+ documentId: params.documentId,
3211
+ revisionKey: params.revisionKey,
3212
+ source: "remoteReconstruction"
3213
+ });
3214
+ await params.parsedStorageConfig.storage.writeSyncProgress({
3215
+ documentId: params.documentId,
3216
+ revisionKey: params.revisionKey,
3217
+ nextChunkPage: totalPages + 1,
3218
+ status: "completed",
3219
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
3220
+ });
3221
+ return {
3222
+ documentId: params.documentId,
3223
+ revisionKey: params.revisionKey,
3224
+ completed: true,
3225
+ commit: synced.commit
3226
+ };
3227
+ }
3228
+ async readRemoteReconstructionChunks(params) {
3229
+ const chunks = [];
3230
+ for (let page = 1; page <= params.totalPages; page += 1) {
3231
+ const object = await params.parsedStorageConfig.storage.readObject({
3232
+ documentId: params.documentId,
3233
+ revisionKey: params.revisionKey,
3234
+ path: getRemoteReconstructionPagePath(page)
3235
+ });
3236
+ if (!object) {
3237
+ return null;
3238
+ }
3239
+ const payload = readJsonObject(object);
3240
+ if (!isDocumentChunkArray(payload.chunks)) {
3241
+ return null;
3242
+ }
3243
+ chunks.push(...payload.chunks);
3244
+ }
3245
+ return chunks;
3246
+ }
3247
+ requireParsedStorageConfig() {
3248
+ if (!this.parsedStorageConfig) {
3249
+ throw new ValidationError("parsed document storage is not configured");
3250
+ }
3251
+ return this.parsedStorageConfig;
3252
+ }
3253
+ async findLocalDocumentByRemoteId(documentId) {
3254
+ const documents = await this.store.listDocuments();
3255
+ return documents.find((document) => document.documentId === documentId);
3256
+ }
3257
+ async getPublishedDocumentJobId(documentId) {
3258
+ const response = await this.client.documents.listChunks(documentId, {
3259
+ page: 1,
3260
+ pageSize: 1,
3261
+ includeAssetUrls: true
3262
+ });
3263
+ if (!response.jobId) {
3264
+ throw new Error(
3265
+ `Cannot sync server document ${documentId}: current published job id was not returned.`
3266
+ );
3267
+ }
3268
+ return response.jobId;
2009
3269
  }
2010
3270
  };
2011
3271
  function normalizeDocumentReference(reference) {
@@ -2024,6 +3284,496 @@ function createLocalDocumentIdForRemote(documentId) {
2024
3284
  const hash = createHash2("sha256").update(documentId).digest("hex").slice(0, 16);
2025
3285
  return `remote_${hash}`;
2026
3286
  }
3287
+ var defaultParsedStorageScheduler = {
3288
+ schedule(task) {
3289
+ void task();
3290
+ }
3291
+ };
3292
+ function getParsedStorageLimits(limits) {
3293
+ return {
3294
+ remotePageSize: clampLimit(
3295
+ limits?.remotePageSize,
3296
+ DEFAULT_REMOTE_SCAN_PAGE_SIZE,
3297
+ MAX_PAGE_SIZE
3298
+ ),
3299
+ maxPagesPerSync: clampLimit(limits?.maxPagesPerSync, DEFAULT_MAX_SYNC_PAGES, 1e3),
3300
+ syncDeadlineMs: clampLimit(limits?.syncDeadlineMs, DEFAULT_SYNC_DEADLINE_MS, 6e4),
3301
+ grepMaxPages: clampLimit(limits?.grepMaxPages, DEFAULT_GREP_MAX_PAGES, 1e3),
3302
+ grepDeadlineMs: clampLimit(limits?.grepDeadlineMs, DEFAULT_GREP_DEADLINE_MS, 6e4),
3303
+ outlineMaxPages: clampLimit(limits?.outlineMaxPages, DEFAULT_OUTLINE_MAX_PAGES, 1e3),
3304
+ outlineDeadlineMs: clampLimit(limits?.outlineDeadlineMs, DEFAULT_OUTLINE_DEADLINE_MS, 6e4)
3305
+ };
3306
+ }
3307
+ function validateReadParams(params) {
3308
+ const usesPagedDisplay = params.page !== void 0 || params.pageSize !== void 0;
3309
+ if (!usesPagedDisplay) {
3310
+ return;
3311
+ }
3312
+ if (params.sectionPath !== void 0 || params.startChunk !== void 0 || params.endChunk !== void 0 || params.chunkId !== void 0) {
3313
+ throw new ValidationError(
3314
+ "page and pageSize cannot be combined with sectionPath, startChunk, endChunk, or chunkId"
3315
+ );
3316
+ }
3317
+ }
3318
+ function hasScanReadFilters(params) {
3319
+ return params.sectionPath !== void 0 || params.startChunk !== void 0 || params.endChunk !== void 0 || params.chunkId !== void 0;
3320
+ }
3321
+ function getReadPageParams(params) {
3322
+ return {
3323
+ page: clampLimit(params.page, 1, Number.MAX_SAFE_INTEGER),
3324
+ pageSize: clampLimit(params.pageSize ?? params.limit, DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE)
3325
+ };
3326
+ }
3327
+ function getRemoteRevisionKey(response) {
3328
+ const revisionKey = response.jobResultId ?? response.jobId;
3329
+ if (!revisionKey) {
3330
+ throw new Error(
3331
+ `Cannot read server document ${response.documentId}: current published revision was not returned.`
3332
+ );
3333
+ }
3334
+ return revisionKey;
3335
+ }
3336
+ function getRemoteReconstructionPagePath(page) {
3337
+ return `${remoteReconstructionPageDirectory}/page-${page}.json`;
3338
+ }
3339
+ function createRemoteKnowledgeDocument(params) {
3340
+ return {
3341
+ localDocumentId: params.documentId,
3342
+ jobId: params.response.jobId ?? params.revisionKey,
3343
+ documentId: params.documentId,
3344
+ namespace: params.response.namespace,
3345
+ sourceFileName: params.documentId,
3346
+ chunkCount: params.response.pagination.total,
3347
+ typeCounts: params.typeCounts ?? countRemoteChunkTypes(params.response.chunks),
3348
+ resultDirectoryPath: `remote:${params.documentId}`,
3349
+ createdAt: /* @__PURE__ */ new Date(0),
3350
+ updatedAt: /* @__PURE__ */ new Date(0)
3351
+ };
3352
+ }
3353
+ function createMinimalParseResultFromRemoteChunks(params) {
3354
+ const chunks = params.chunks.map(toMinimalChunkFromRemoteChunk);
3355
+ const typeCounts = countRemoteChunkTypes(params.chunks);
3356
+ return {
3357
+ manifest: {
3358
+ version: "2.0",
3359
+ jobId: params.response.jobId ?? params.revisionKey,
3360
+ sourceFileName: params.documentId,
3361
+ statistics: {
3362
+ totalChunks: chunks.length,
3363
+ textChunks: typeCounts.text,
3364
+ imageChunks: typeCounts.image,
3365
+ tableChunks: typeCounts.table,
3366
+ pageChunks: typeCounts.page
3367
+ }
3368
+ },
3369
+ chunks,
3370
+ rawZip: Buffer.from(""),
3371
+ namespace: params.response.namespace,
3372
+ documentId: params.documentId,
3373
+ textChunks: chunks.filter(
3374
+ (chunk) => chunk.type === "text"
3375
+ ),
3376
+ imageChunks: chunks.filter(
3377
+ (chunk) => chunk.type === "image"
3378
+ ),
3379
+ tableChunks: chunks.filter(
3380
+ (chunk) => chunk.type === "table"
3381
+ ),
3382
+ pageChunks: chunks.filter(
3383
+ (chunk) => chunk.type === "page"
3384
+ ),
3385
+ jobId: params.response.jobId ?? params.revisionKey,
3386
+ statistics: {
3387
+ totalChunks: chunks.length,
3388
+ textChunks: typeCounts.text,
3389
+ imageChunks: typeCounts.image,
3390
+ tableChunks: typeCounts.table,
3391
+ pageChunks: typeCounts.page
3392
+ },
3393
+ getChunk: (chunkId) => chunks.find((chunk) => chunk.chunkId === chunkId),
3394
+ save: () => Promise.reject(new Error("Minimal remote reconstruction cannot be saved directly."))
3395
+ };
3396
+ }
3397
+ function toMinimalChunkFromRemoteChunk(chunk) {
3398
+ const baseChunk = {
3399
+ chunkId: chunk.chunkId,
3400
+ contentSource: chunk.contentSource ?? void 0,
3401
+ content: chunk.content ?? "",
3402
+ path: chunk.sourceChunkPath ?? chunk.sectionPath ?? "",
3403
+ metadata: cloneMetadata(chunk.metadata)
3404
+ };
3405
+ if (chunk.chunkType === "image") {
3406
+ return {
3407
+ ...baseChunk,
3408
+ type: "image",
3409
+ filePath: chunk.filePath ?? "",
3410
+ assetUrl: chunk.assetUrl ?? void 0,
3411
+ data: Buffer.alloc(0),
3412
+ format: inferContentTypeFromPath2(chunk.filePath ?? "").replace("image/", ""),
3413
+ save: () => Promise.resolve(chunk.filePath ?? "")
3414
+ };
3415
+ }
3416
+ if (chunk.chunkType === "table") {
3417
+ return {
3418
+ ...baseChunk,
3419
+ type: "table",
3420
+ filePath: chunk.filePath ?? "",
3421
+ assetUrl: chunk.assetUrl ?? void 0,
3422
+ html: chunk.content ?? "",
3423
+ save: () => Promise.resolve(chunk.filePath ?? "")
3424
+ };
3425
+ }
3426
+ if (chunk.chunkType === "page") {
3427
+ return {
3428
+ ...baseChunk,
3429
+ type: "page"
3430
+ };
3431
+ }
3432
+ return {
3433
+ ...baseChunk,
3434
+ type: "text"
3435
+ };
3436
+ }
3437
+ function createStoredKnowledgeDocument(params) {
3438
+ return {
3439
+ localDocumentId: params.documentId,
3440
+ jobId: params.storedResult.manifest.jobId || params.revisionKey,
3441
+ documentId: params.documentId,
3442
+ sourceFileName: params.storedResult.manifest.sourceFileName,
3443
+ chunkCount: params.storedResult.chunks.length,
3444
+ typeCounts: params.storedResult.typeCounts,
3445
+ resultDirectoryPath: `parsed-storage:${params.documentId}`,
3446
+ createdAt: new Date(params.storedResult.commit.committedAt),
3447
+ updatedAt: new Date(params.storedResult.commit.committedAt)
3448
+ };
3449
+ }
3450
+ function toRemoteReadChunk(chunk) {
3451
+ const metadata = cloneMetadata(chunk.metadata);
3452
+ return {
3453
+ position: toChunkPosition(chunk),
3454
+ chunkId: chunk.chunkId,
3455
+ chunkType: chunk.chunkType,
3456
+ contentSource: chunk.contentSource ?? void 0,
3457
+ content: chunk.content ?? "",
3458
+ readableContent: getReadableRemoteChunkContent(chunk, metadata),
3459
+ sectionPath: chunk.sectionPath ?? "",
3460
+ sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? "",
3461
+ filePath: chunk.filePath ?? void 0,
3462
+ assetUrl: chunk.assetUrl ?? void 0,
3463
+ pageNumbers: getChunkPageNumbers(metadata),
3464
+ metadata
3465
+ };
3466
+ }
3467
+ function getReadableRemoteChunkContent(chunk, metadata) {
3468
+ if (chunk.chunkType !== "page") {
3469
+ return chunk.content ?? "";
3470
+ }
3471
+ const summary = metadata.summary;
3472
+ return typeof summary === "string" && summary.trim().length > 0 ? summary : chunk.content ?? "";
3473
+ }
3474
+ function toIndexedRemoteChunk(chunk) {
3475
+ const readChunk = toRemoteReadChunk(chunk);
3476
+ return toIndexedReadChunk(readChunk);
3477
+ }
3478
+ function parseStoredResultChunks(params) {
3479
+ const rawChunks = extractStoredChunkRecords(params.payload);
3480
+ return rawChunks.map((record, index) => {
3481
+ const metadata = readRecordMetadata(record);
3482
+ const chunkType = readRecordString(record, "type", "chunkType", "chunk_type");
3483
+ const content = readRecordString(record, "content") ?? "";
3484
+ const sourceChunkPath = readRecordString(record, "path", "sourceChunkPath", "source_chunk_path") ?? "";
3485
+ const chunk = {
3486
+ position: index + 1,
3487
+ chunkId: readRecordString(record, "chunkId", "chunk_id") ?? "",
3488
+ chunkType: isKnowledgeChunkType(chunkType) ? chunkType : "text",
3489
+ contentSource: readRecordString(record, "contentSource", "content_source"),
3490
+ content,
3491
+ readableContent: chunkType === "page" && typeof metadata.summary === "string" && metadata.summary.trim().length > 0 ? metadata.summary : content,
3492
+ sectionPath: normalizeSectionPath(sourceChunkPath, params.sourceFileName),
3493
+ sourceChunkPath,
3494
+ filePath: readRecordString(record, "filePath", "file_path") ?? (typeof metadata.filePath === "string" ? metadata.filePath : void 0),
3495
+ assetUrl: readRecordString(record, "assetUrl", "asset_url"),
3496
+ pageNumbers: getChunkPageNumbers(metadata),
3497
+ metadata
3498
+ };
3499
+ return toIndexedReadChunk(chunk);
3500
+ });
3501
+ }
3502
+ function extractStoredChunkRecords(payload) {
3503
+ if (Array.isArray(payload)) {
3504
+ return payload.filter(isRecord2);
3505
+ }
3506
+ if (isRecord2(payload) && Array.isArray(payload.chunks)) {
3507
+ return payload.chunks.filter(isRecord2);
3508
+ }
3509
+ return [];
3510
+ }
3511
+ function readRecordString(record, ...keys) {
3512
+ for (const key of keys) {
3513
+ const value = record[key];
3514
+ if (typeof value === "string") return value;
3515
+ }
3516
+ return void 0;
3517
+ }
3518
+ function readRecordNumber(record, ...keys) {
3519
+ for (const key of keys) {
3520
+ const value = record[key];
3521
+ if (typeof value === "number") return value;
3522
+ }
3523
+ return void 0;
3524
+ }
3525
+ function readRecordMetadata(record) {
3526
+ const metadata = record.metadata;
3527
+ return isRecord2(metadata) ? keysToCamel(metadata) : {};
3528
+ }
3529
+ function isKnowledgeChunkType(value) {
3530
+ return value === "text" || value === "image" || value === "table" || value === "page";
3531
+ }
3532
+ function toIndexedReadChunk(chunk) {
3533
+ return {
3534
+ source: createSyntheticChunk(chunk),
3535
+ position: chunk.position,
3536
+ chunkId: chunk.chunkId,
3537
+ chunkType: chunk.chunkType,
3538
+ contentSource: chunk.contentSource,
3539
+ content: chunk.content,
3540
+ readableContent: chunk.readableContent,
3541
+ sectionPath: chunk.sectionPath,
3542
+ sourceChunkPath: chunk.sourceChunkPath,
3543
+ filePath: chunk.filePath,
3544
+ assetUrl: chunk.assetUrl,
3545
+ pageNumbers: chunk.pageNumbers,
3546
+ metadata: chunk.metadata
3547
+ };
3548
+ }
3549
+ function createSyntheticChunk(chunk) {
3550
+ return {
3551
+ chunkId: chunk.chunkId,
3552
+ type: chunk.chunkType,
3553
+ contentSource: chunk.contentSource,
3554
+ content: chunk.content,
3555
+ path: chunk.sourceChunkPath,
3556
+ metadata: chunk.metadata
3557
+ };
3558
+ }
3559
+ function toChunkPosition(chunk) {
3560
+ return chunk ? chunk.sortOrder + 1 : 1;
3561
+ }
3562
+ function buildFlatSectionsFromIndexedChunks(chunks) {
3563
+ const byPath = /* @__PURE__ */ new Map();
3564
+ for (const chunk of chunks) {
3565
+ const path4 = chunk.sectionPath || chunk.sourceChunkPath;
3566
+ const existing = byPath.get(path4);
3567
+ if (existing) {
3568
+ addChunkToSection(existing, chunk);
3569
+ } else {
3570
+ byPath.set(path4, createSectionFromChunk(path4, chunk));
3571
+ }
3572
+ }
3573
+ return [...byPath.values()].sort(compareSections);
3574
+ }
3575
+ function matchesRemoteReadScope(chunk, params) {
3576
+ return matchesReadScope(toIndexedRemoteChunk(chunk), params);
3577
+ }
3578
+ function getNextChunkPosition(chunks, selected) {
3579
+ const lastSelected = selected[selected.length - 1];
3580
+ if (!lastSelected || selected.length >= chunks.length) {
3581
+ return void 0;
3582
+ }
3583
+ return chunks[chunks.indexOf(lastSelected) + 1]?.position;
3584
+ }
3585
+ async function applyStoredAssetUrlsToReadChunks(params) {
3586
+ return Promise.all(
3587
+ params.chunks.map(
3588
+ (chunk) => applyStoredAssetUrlsToReadChunk({
3589
+ ...params,
3590
+ chunk
3591
+ })
3592
+ )
3593
+ );
3594
+ }
3595
+ async function applyStoredAssetUrlsToReadChunk(params) {
3596
+ const metadata = cloneMetadata(params.chunk.metadata);
3597
+ const storedAssetUrl = await getStoredObjectUrl({
3598
+ storageConfig: params.storageConfig,
3599
+ documentId: params.documentId,
3600
+ revisionKey: params.revisionKey,
3601
+ path: params.chunk.filePath
3602
+ });
3603
+ const pageAssets = await applyStoredPageAssetUrls({
3604
+ storageConfig: params.storageConfig,
3605
+ documentId: params.documentId,
3606
+ revisionKey: params.revisionKey,
3607
+ metadata
3608
+ });
3609
+ if (pageAssets) {
3610
+ metadata.pageAssets = pageAssets;
3611
+ delete metadata.page_assets;
3612
+ }
3613
+ return {
3614
+ ...params.chunk,
3615
+ assetUrl: storedAssetUrl ?? params.chunk.assetUrl,
3616
+ metadata
3617
+ };
3618
+ }
3619
+ async function applyStoredPageAssetUrls(params) {
3620
+ const pageAssets = getStoredPageAssets(params.metadata);
3621
+ if (!pageAssets) {
3622
+ return void 0;
3623
+ }
3624
+ return Promise.all(
3625
+ pageAssets.map(async (pageAsset) => {
3626
+ if (!isRecord2(pageAsset)) {
3627
+ return pageAsset;
3628
+ }
3629
+ const artifactRef = readRecordString(pageAsset, "artifactRef", "artifact_ref");
3630
+ if (typeof artifactRef !== "string") {
3631
+ return pageAsset;
3632
+ }
3633
+ const storedAssetUrl = await getStoredObjectUrl({
3634
+ storageConfig: params.storageConfig,
3635
+ documentId: params.documentId,
3636
+ revisionKey: params.revisionKey,
3637
+ path: artifactRef
3638
+ });
3639
+ if (storedAssetUrl) {
3640
+ return {
3641
+ ...pageAsset,
3642
+ artifactRef,
3643
+ assetUrl: storedAssetUrl
3644
+ };
3645
+ }
3646
+ return pageAsset;
3647
+ })
3648
+ );
3649
+ }
3650
+ function getStoredPageAssets(metadata) {
3651
+ const pageAssets = toUnknownArray(metadata.pageAssets) ?? toUnknownArray(metadata.page_assets) ?? void 0;
3652
+ return pageAssets?.map(normalizeStoredPageAsset);
3653
+ }
3654
+ function normalizeStoredPageAsset(pageAsset) {
3655
+ if (!isRecord2(pageAsset)) {
3656
+ return pageAsset;
3657
+ }
3658
+ const normalizedPageAsset = { ...pageAsset };
3659
+ delete normalizedPageAsset.page_num;
3660
+ delete normalizedPageAsset.artifact_ref;
3661
+ delete normalizedPageAsset.asset_url;
3662
+ delete normalizedPageAsset.content_type;
3663
+ const pageNum = readRecordNumber(pageAsset, "pageNum", "page_num");
3664
+ const artifactRef = readRecordString(pageAsset, "artifactRef", "artifact_ref");
3665
+ const assetUrl = readRecordString(pageAsset, "assetUrl", "asset_url");
3666
+ const contentType = readRecordString(pageAsset, "contentType", "content_type");
3667
+ return {
3668
+ ...normalizedPageAsset,
3669
+ ...pageNum === void 0 ? {} : { pageNum },
3670
+ ...artifactRef === void 0 ? {} : { artifactRef },
3671
+ ...assetUrl === void 0 ? {} : { assetUrl },
3672
+ ...contentType === void 0 ? {} : { contentType }
3673
+ };
3674
+ }
3675
+ async function getStoredObjectUrl(params) {
3676
+ if (!params.path) {
3677
+ return void 0;
3678
+ }
3679
+ return await params.storageConfig.storage.getObjectUrl?.({
3680
+ documentId: params.documentId,
3681
+ revisionKey: params.revisionKey,
3682
+ path: params.path
3683
+ }) ?? void 0;
3684
+ }
3685
+ function countRemoteChunkTypes(chunks) {
3686
+ return chunks.reduce(
3687
+ (counts, chunk) => {
3688
+ counts[chunk.chunkType] += 1;
3689
+ return counts;
3690
+ },
3691
+ { text: 0, image: 0, table: 0, page: 0 }
3692
+ );
3693
+ }
3694
+ function createContinuationCursor(cursor) {
3695
+ return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
3696
+ }
3697
+ function createGrepContinuationCursor(params) {
3698
+ if (params.matchIndex + 1 < params.chunkMatchCount) {
3699
+ return createContinuationCursor({
3700
+ documentId: params.documentId,
3701
+ revisionKey: params.revisionKey,
3702
+ nextPage: params.response.pagination.page,
3703
+ nextChunkIndex: params.chunkIndex,
3704
+ nextMatchIndex: params.matchIndex + 1
3705
+ });
3706
+ }
3707
+ if (params.chunkIndex + 1 < params.response.chunks.length) {
3708
+ return createContinuationCursor({
3709
+ documentId: params.documentId,
3710
+ revisionKey: params.revisionKey,
3711
+ nextPage: params.response.pagination.page,
3712
+ nextChunkIndex: params.chunkIndex + 1,
3713
+ nextMatchIndex: 0
3714
+ });
3715
+ }
3716
+ if (params.response.pagination.page < params.response.pagination.totalPages) {
3717
+ return createContinuationCursor({
3718
+ documentId: params.documentId,
3719
+ revisionKey: params.revisionKey,
3720
+ nextPage: params.response.pagination.page + 1,
3721
+ nextChunkIndex: 0,
3722
+ nextMatchIndex: 0
3723
+ });
3724
+ }
3725
+ return void 0;
3726
+ }
3727
+ function parseContinuationCursor(value) {
3728
+ if (!value) {
3729
+ return null;
3730
+ }
3731
+ try {
3732
+ const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
3733
+ if (!isRecord2(parsed)) {
3734
+ return null;
3735
+ }
3736
+ const documentId = parsed.documentId;
3737
+ const revisionKey = parsed.revisionKey;
3738
+ const nextPage = parsed.nextPage;
3739
+ const nextChunkIndex = parsed.nextChunkIndex;
3740
+ const nextMatchIndex = parsed.nextMatchIndex;
3741
+ if (typeof documentId !== "string" || typeof revisionKey !== "string" || typeof nextPage !== "number" || !Number.isInteger(nextPage) || nextPage < 1 || !isOptionalNonNegativeInteger(nextChunkIndex) || !isOptionalNonNegativeInteger(nextMatchIndex)) {
3742
+ return null;
3743
+ }
3744
+ return {
3745
+ documentId,
3746
+ revisionKey,
3747
+ nextPage,
3748
+ nextChunkIndex,
3749
+ nextMatchIndex
3750
+ };
3751
+ } catch {
3752
+ return null;
3753
+ }
3754
+ }
3755
+ function isOptionalNonNegativeInteger(value) {
3756
+ return value === void 0 || typeof value === "number" && Number.isInteger(value) && value >= 0;
3757
+ }
3758
+ function cloneMetadata(metadata) {
3759
+ return { ...metadata };
3760
+ }
3761
+ function toUnknownArray(value) {
3762
+ return Array.isArray(value) ? value : null;
3763
+ }
3764
+ function inferContentTypeFromPath2(filePath) {
3765
+ const extension = filePath.split(".").at(-1)?.toLowerCase();
3766
+ if (extension === "png") return "image/png";
3767
+ if (extension === "jpg" || extension === "jpeg") return "image/jpeg";
3768
+ if (extension === "html" || extension === "htm") return "text/html; charset=utf-8";
3769
+ return "application/octet-stream";
3770
+ }
3771
+ function isRecord2(value) {
3772
+ return typeof value === "object" && value !== null && !Array.isArray(value);
3773
+ }
3774
+ function isDocumentChunkArray(value) {
3775
+ return Array.isArray(value) && value.every(isRecord2);
3776
+ }
2027
3777
  function indexChunks(result) {
2028
3778
  return result.chunks.map((chunk, index) => {
2029
3779
  const filePath = getChunkFilePath2(chunk);
@@ -2039,6 +3789,7 @@ function indexChunks(result) {
2039
3789
  sectionPath: normalizeSectionPath(chunk.path, result.manifest.sourceFileName),
2040
3790
  sourceChunkPath: chunk.path,
2041
3791
  filePath,
3792
+ assetUrl: getChunkAssetUrl(chunk),
2042
3793
  pageNumbers: getChunkPageNumbers(chunk.metadata),
2043
3794
  metadata: chunk.metadata
2044
3795
  };
@@ -2052,12 +3803,11 @@ function getReadableChunkContent(chunk) {
2052
3803
  return typeof summary === "string" && summary.trim().length > 0 ? summary : chunk.content;
2053
3804
  }
2054
3805
  function getChunkPageNumbers(metadata) {
2055
- const value = metadata.pageNums;
2056
- if (!Array.isArray(value)) {
2057
- return void 0;
2058
- }
2059
- const pageNumbers = value.filter(
2060
- (pageNumber) => Number.isInteger(pageNumber) && pageNumber > 0
3806
+ const values = [metadata.pageNums, metadata.page_nums].filter(Array.isArray);
3807
+ const pageNumbers = values.flatMap(
3808
+ (value) => value.filter(
3809
+ (pageNumber) => Number.isInteger(pageNumber) && pageNumber > 0
3810
+ )
2061
3811
  );
2062
3812
  return pageNumbers.length > 0 ? [...new Set(pageNumbers)].sort((left, right) => left - right) : void 0;
2063
3813
  }
@@ -2068,14 +3818,17 @@ function getChunkFilePath2(chunk) {
2068
3818
  const filePath = chunk.metadata.filePath;
2069
3819
  return typeof filePath === "string" ? filePath : void 0;
2070
3820
  }
2071
- function normalizeSectionPath(path3, sourceFileName) {
2072
- if (!path3) {
3821
+ function getChunkAssetUrl(chunk) {
3822
+ return chunk.type === "image" || chunk.type === "table" ? chunk.assetUrl : void 0;
3823
+ }
3824
+ function normalizeSectionPath(path4, sourceFileName) {
3825
+ if (!path4) {
2073
3826
  return "";
2074
3827
  }
2075
- if (path3.startsWith("images/") || path3.startsWith("tables/")) {
2076
- return path3;
3828
+ if (path4.startsWith("images/") || path4.startsWith("tables/")) {
3829
+ return path4;
2077
3830
  }
2078
- const parts = path3.split("/").filter(Boolean);
3831
+ const parts = path4.split("/").filter(Boolean);
2079
3832
  if (sourceFileName) {
2080
3833
  const fileNameIndex = parts.indexOf(sourceFileName);
2081
3834
  if (fileNameIndex >= 0) {
@@ -2097,12 +3850,12 @@ function buildFlatSections(result, chunks) {
2097
3850
  }
2098
3851
  const byPath = /* @__PURE__ */ new Map();
2099
3852
  for (const chunk of chunks) {
2100
- const path3 = chunk.sectionPath || chunk.sourceChunkPath;
2101
- const existing = byPath.get(path3);
3853
+ const path4 = chunk.sectionPath || chunk.sourceChunkPath;
3854
+ const existing = byPath.get(path4);
2102
3855
  if (existing) {
2103
3856
  addChunkToSection(existing, chunk);
2104
3857
  } else {
2105
- byPath.set(path3, createSectionFromChunk(path3, chunk));
3858
+ byPath.set(path4, createSectionFromChunk(path4, chunk));
2106
3859
  }
2107
3860
  }
2108
3861
  return [...byPath.values()].sort(compareSections);
@@ -2236,6 +3989,7 @@ function toReadChunk(chunk) {
2236
3989
  sectionPath: chunk.sectionPath,
2237
3990
  sourceChunkPath: chunk.sourceChunkPath,
2238
3991
  filePath: chunk.filePath,
3992
+ assetUrl: chunk.assetUrl,
2239
3993
  pageNumbers: chunk.pageNumbers,
2240
3994
  metadata: chunk.metadata
2241
3995
  };
@@ -2412,7 +4166,12 @@ var Knowhere = class {
2412
4166
  const result = await this.jobs.load(jobResult, {
2413
4167
  verifyChecksum: params.verifyChecksum
2414
4168
  });
2415
- return enrichParseResult(result, jobResult);
4169
+ const enrichedResult = enrichParseResult(result, {
4170
+ namespace: jobResult.namespace,
4171
+ documentId: jobResult.documentId ?? params.documentId
4172
+ });
4173
+ const storedResult = await storeParseResultAssets(enrichedResult, params.storageAdapter);
4174
+ return storedResult.result;
2416
4175
  }
2417
4176
  /**
2418
4177
  * Start a parse job and return immediately after the URL job is created or
@@ -2455,12 +4214,167 @@ var Knowhere = class {
2455
4214
  return job;
2456
4215
  }
2457
4216
  };
4217
+
4218
+ // src/storage/disk-parsed-document-storage.ts
4219
+ import { createHash as createHash3 } from "crypto";
4220
+ import { promises as fs4 } from "fs";
4221
+ import path3 from "path";
4222
+ var syncProgressPath = ".knowhere-sdk/sync-progress.json";
4223
+ var DiskParsedDocumentStorage = class {
4224
+ rootDirectory;
4225
+ constructor(rootDirectory) {
4226
+ this.rootDirectory = rootDirectory;
4227
+ }
4228
+ async readObject(params) {
4229
+ const objectPath = this.getObjectPath(params);
4230
+ try {
4231
+ const body = await fs4.readFile(objectPath);
4232
+ const metadata = await this.readObjectMetadata(objectPath);
4233
+ return {
4234
+ ...params,
4235
+ body,
4236
+ contentType: metadata?.contentType,
4237
+ contentLength: body.byteLength,
4238
+ metadata: metadata?.metadata,
4239
+ url: toFileUrl(objectPath)
4240
+ };
4241
+ } catch (error) {
4242
+ if (isMissingFileError3(error)) {
4243
+ return null;
4244
+ }
4245
+ throw error;
4246
+ }
4247
+ }
4248
+ async writeObject(params) {
4249
+ const objectPath = this.getObjectPath(params);
4250
+ await fs4.mkdir(path3.dirname(objectPath), { recursive: true });
4251
+ await fs4.writeFile(objectPath, params.body);
4252
+ await this.writeObjectMetadata(objectPath, {
4253
+ contentType: params.contentType,
4254
+ metadata: params.metadata
4255
+ });
4256
+ return {
4257
+ documentId: params.documentId,
4258
+ revisionKey: params.revisionKey,
4259
+ path: params.path,
4260
+ url: toFileUrl(objectPath)
4261
+ };
4262
+ }
4263
+ async headObject(params) {
4264
+ const objectPath = this.getObjectPath(params);
4265
+ try {
4266
+ const stat = await fs4.stat(objectPath);
4267
+ const metadata = await this.readObjectMetadata(objectPath);
4268
+ return {
4269
+ ...params,
4270
+ contentType: metadata?.contentType,
4271
+ contentLength: stat.size,
4272
+ metadata: metadata?.metadata,
4273
+ url: toFileUrl(objectPath)
4274
+ };
4275
+ } catch (error) {
4276
+ if (isMissingFileError3(error)) {
4277
+ return null;
4278
+ }
4279
+ throw error;
4280
+ }
4281
+ }
4282
+ async getObjectUrl(params) {
4283
+ const head = await this.headObject(params);
4284
+ return head?.url ?? null;
4285
+ }
4286
+ async deletePrefix(params) {
4287
+ await fs4.rm(this.getRevisionDirectory(params.documentId, params.revisionKey), {
4288
+ recursive: true,
4289
+ force: true
4290
+ });
4291
+ }
4292
+ async readSyncProgress(params) {
4293
+ return this.readOptionalJson({
4294
+ ...params,
4295
+ path: syncProgressPath
4296
+ });
4297
+ }
4298
+ async writeSyncProgress(params) {
4299
+ await this.writeObject({
4300
+ documentId: params.documentId,
4301
+ revisionKey: params.revisionKey,
4302
+ path: syncProgressPath,
4303
+ body: Buffer.from(JSON.stringify(params, null, 2), "utf8"),
4304
+ contentType: "application/json; charset=utf-8"
4305
+ });
4306
+ }
4307
+ getObjectPath(params) {
4308
+ const revisionDirectory = this.getRevisionDirectory(params.documentId, params.revisionKey);
4309
+ const objectPath = path3.resolve(revisionDirectory, normalizeRelativeStoragePath(params.path));
4310
+ if (!isPathInsideDirectory2(objectPath, revisionDirectory)) {
4311
+ throw new Error(`Parsed document object path resolves outside storage: ${params.path}`);
4312
+ }
4313
+ return objectPath;
4314
+ }
4315
+ getRevisionDirectory(documentId, revisionKey) {
4316
+ return path3.join(this.rootDirectory, hashPathPart(documentId), hashPathPart(revisionKey));
4317
+ }
4318
+ async readOptionalJson(params) {
4319
+ const object = await this.readObject(params);
4320
+ if (!object) {
4321
+ return null;
4322
+ }
4323
+ return JSON.parse(Buffer.from(object.body).toString("utf8"));
4324
+ }
4325
+ async readObjectMetadata(filePath) {
4326
+ try {
4327
+ return JSON.parse(
4328
+ await fs4.readFile(getMetadataPath(filePath), "utf8")
4329
+ );
4330
+ } catch (error) {
4331
+ if (isMissingFileError3(error)) {
4332
+ return null;
4333
+ }
4334
+ throw error;
4335
+ }
4336
+ }
4337
+ async writeObjectMetadata(filePath, metadata) {
4338
+ await fs4.writeFile(getMetadataPath(filePath), JSON.stringify(metadata, null, 2), {
4339
+ encoding: "utf8",
4340
+ flag: "w"
4341
+ });
4342
+ }
4343
+ };
4344
+ function getMetadataPath(filePath) {
4345
+ return `${filePath}.metadata.json`;
4346
+ }
4347
+ function hashPathPart(value) {
4348
+ const normalized = value.trim();
4349
+ if (normalized.length === 0) {
4350
+ throw new Error("Parsed document storage path part must be a non-empty value.");
4351
+ }
4352
+ return createHash3("sha256").update(normalized).digest("hex").slice(0, 24);
4353
+ }
4354
+ function normalizeRelativeStoragePath(value) {
4355
+ const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
4356
+ if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
4357
+ throw new Error(`Invalid parsed storage path: ${value}`);
4358
+ }
4359
+ return normalized;
4360
+ }
4361
+ function isPathInsideDirectory2(filePath, directory) {
4362
+ const relative = path3.relative(directory, filePath);
4363
+ return relative === "" || !relative.startsWith("..") && !path3.isAbsolute(relative);
4364
+ }
4365
+ function toFileUrl(filePath) {
4366
+ return `file://${filePath}`;
4367
+ }
4368
+ function isMissingFileError3(error) {
4369
+ return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
4370
+ }
2458
4371
  export {
2459
4372
  APIError,
2460
4373
  AuthenticationError,
2461
4374
  BadRequestError,
2462
4375
  ChecksumError,
2463
4376
  ConflictError,
4377
+ DiskParsedDocumentStorage,
2464
4378
  Documents,
2465
4379
  GatewayTimeoutError,
2466
4380
  InternalServerError,
@@ -2482,5 +4396,7 @@ export {
2482
4396
  TimeoutError,
2483
4397
  VERSION,
2484
4398
  ValidationError,
2485
- Knowhere as default
4399
+ Knowhere as default,
4400
+ storeParseResultAssets,
4401
+ syncParseResultToParsedDocumentStorage
2486
4402
  };