deeplake 0.3.16 → 0.3.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/dist/browser/{browser/client.d.ts → client.d.ts} +2 -2
  2. package/dist/browser/{browser/client.js → client.js} +8 -8
  3. package/dist/browser/{browser/index.d.ts → index.d.ts} +12 -12
  4. package/dist/browser/{browser/index.js → index.js} +8 -8
  5. package/dist/browser/shared/credentials.d.ts.map +1 -1
  6. package/dist/browser/shared/credentials.js +4 -6
  7. package/dist/browser/shared/credentials.js.map +1 -1
  8. package/dist/browser/shared/database.d.ts +3 -1
  9. package/dist/browser/shared/database.d.ts.map +1 -1
  10. package/dist/browser/shared/dlref.d.ts +3 -0
  11. package/dist/browser/shared/dlref.d.ts.map +1 -1
  12. package/dist/browser/shared/dlref.js +3 -0
  13. package/dist/browser/shared/dlref.js.map +1 -1
  14. package/dist/browser/shared/schema.d.ts.map +1 -1
  15. package/dist/browser/shared/schema.js +31 -20
  16. package/dist/browser/shared/schema.js.map +1 -1
  17. package/dist/browser/shared/token.d.ts.map +1 -1
  18. package/dist/browser/shared/token.js +1 -17
  19. package/dist/browser/shared/token.js.map +1 -1
  20. package/dist/browser/{browser/storage.d.ts.map → storage.d.ts.map} +1 -1
  21. package/dist/browser/{browser/storage.js → storage.js} +20 -33
  22. package/dist/browser/storage.js.map +1 -0
  23. package/dist/browser/{browser/wasm.js → wasm.js} +1 -1
  24. package/dist/node/client.d.ts +107 -0
  25. package/dist/node/client.d.ts.map +1 -0
  26. package/dist/node/{node/client.js → client.js} +138 -112
  27. package/dist/node/client.js.map +1 -0
  28. package/dist/node/formats/coco.d.ts +62 -0
  29. package/dist/node/formats/coco.d.ts.map +1 -0
  30. package/dist/node/{node/formats → formats}/coco.js +94 -8
  31. package/dist/node/formats/coco.js.map +1 -0
  32. package/dist/node/{node/formats → formats}/coco_panoptic.d.ts +1 -1
  33. package/dist/node/{node/formats → formats}/coco_panoptic.d.ts.map +1 -1
  34. package/dist/node/{node/formats → formats}/coco_panoptic.js +1 -0
  35. package/dist/node/formats/coco_panoptic.js.map +1 -0
  36. package/dist/node/formats/lerobot.d.ts +84 -0
  37. package/dist/node/formats/lerobot.d.ts.map +1 -0
  38. package/dist/node/{node/formats → formats}/lerobot.js +61 -1
  39. package/dist/node/formats/lerobot.js.map +1 -0
  40. package/dist/node/{node/index.d.ts → index.d.ts} +12 -12
  41. package/dist/node/{node/index.js → index.js} +8 -8
  42. package/dist/node/normalizers/binary.d.ts +3 -0
  43. package/dist/node/{node/normalizers → normalizers}/binary.js +1 -1
  44. package/dist/node/{node/normalizers → normalizers}/binary.js.map +1 -1
  45. package/dist/node/{node/normalizers → normalizers}/dict.d.ts +1 -1
  46. package/dist/node/{node/normalizers → normalizers}/files.d.ts +1 -1
  47. package/dist/node/{node/normalizers → normalizers}/files.js +1 -1
  48. package/dist/node/normalizers/huggingface.d.ts +23 -0
  49. package/dist/node/normalizers/huggingface.d.ts.map +1 -0
  50. package/dist/node/{node/normalizers → normalizers}/huggingface.js +29 -0
  51. package/dist/node/normalizers/huggingface.js.map +1 -0
  52. package/dist/node/normalizers/image.d.ts +3 -0
  53. package/dist/node/{node/normalizers → normalizers}/image.js +1 -1
  54. package/dist/node/{node/normalizers → normalizers}/image.js.map +1 -1
  55. package/dist/node/normalizers/pdf.d.ts +3 -0
  56. package/dist/node/{node/normalizers → normalizers}/pdf.js +2 -2
  57. package/dist/node/{node/normalizers → normalizers}/pdf.js.map +1 -1
  58. package/dist/node/{node/normalizers → normalizers}/text.d.ts +1 -1
  59. package/dist/node/{node/normalizers → normalizers}/text.js +1 -1
  60. package/dist/node/{node/normalizers → normalizers}/text.js.map +1 -1
  61. package/dist/node/{node/normalizers → normalizers}/video.d.ts +1 -1
  62. package/dist/node/{node/normalizers → normalizers}/video.js +2 -2
  63. package/dist/node/{node/normalizers → normalizers}/video.js.map +1 -1
  64. package/dist/node/shared/credentials.d.ts.map +1 -1
  65. package/dist/node/shared/credentials.js +4 -6
  66. package/dist/node/shared/credentials.js.map +1 -1
  67. package/dist/node/shared/database.d.ts +3 -1
  68. package/dist/node/shared/database.d.ts.map +1 -1
  69. package/dist/node/shared/dlref.d.ts +3 -0
  70. package/dist/node/shared/dlref.d.ts.map +1 -1
  71. package/dist/node/shared/dlref.js +3 -0
  72. package/dist/node/shared/dlref.js.map +1 -1
  73. package/dist/node/shared/schema.d.ts.map +1 -1
  74. package/dist/node/shared/schema.js +31 -20
  75. package/dist/node/shared/schema.js.map +1 -1
  76. package/dist/node/shared/token.d.ts.map +1 -1
  77. package/dist/node/shared/token.js +1 -17
  78. package/dist/node/shared/token.js.map +1 -1
  79. package/dist/node/{node/storage.d.ts → storage.d.ts} +5 -0
  80. package/dist/node/storage.d.ts.map +1 -0
  81. package/dist/node/{node/storage.js → storage.js} +137 -16
  82. package/dist/node/storage.js.map +1 -0
  83. package/dist/node/{node/wasm.js → wasm.js} +1 -1
  84. package/package.json +17 -13
  85. package/wasm/browser/deeplake_browser.js +1 -1
  86. package/wasm/browser/deeplake_browser.wasm +0 -0
  87. package/wasm/node/deeplake_node.js +1 -1
  88. package/wasm/node/deeplake_node.wasm +0 -0
  89. package/dist/browser/browser/storage.js.map +0 -1
  90. package/dist/node/node/client.d.ts +0 -49
  91. package/dist/node/node/client.d.ts.map +0 -1
  92. package/dist/node/node/client.js.map +0 -1
  93. package/dist/node/node/formats/coco.d.ts +0 -24
  94. package/dist/node/node/formats/coco.d.ts.map +0 -1
  95. package/dist/node/node/formats/coco.js.map +0 -1
  96. package/dist/node/node/formats/coco_panoptic.js.map +0 -1
  97. package/dist/node/node/formats/lerobot.d.ts +0 -42
  98. package/dist/node/node/formats/lerobot.d.ts.map +0 -1
  99. package/dist/node/node/formats/lerobot.js.map +0 -1
  100. package/dist/node/node/normalizers/binary.d.ts +0 -3
  101. package/dist/node/node/normalizers/huggingface.d.ts +0 -3
  102. package/dist/node/node/normalizers/huggingface.d.ts.map +0 -1
  103. package/dist/node/node/normalizers/huggingface.js.map +0 -1
  104. package/dist/node/node/normalizers/image.d.ts +0 -3
  105. package/dist/node/node/normalizers/pdf.d.ts +0 -3
  106. package/dist/node/node/storage.d.ts.map +0 -1
  107. package/dist/node/node/storage.js.map +0 -1
  108. package/dist/wasm/node/deeplake_node.js +0 -3
  109. package/dist/wasm/node/deeplake_node.wasm +0 -0
  110. package/dist/wasm/node/deeplake_node.worker.js +0 -5
  111. package/wasm/browser/deeplake_browser.worker.js +0 -5
  112. package/wasm/node/deeplake_node.worker.js +0 -5
  113. /package/dist/browser/{browser/client.d.ts.map → client.d.ts.map} +0 -0
  114. /package/dist/browser/{browser/client.js.map → client.js.map} +0 -0
  115. /package/dist/browser/{browser/index.d.ts.map → index.d.ts.map} +0 -0
  116. /package/dist/browser/{browser/index.js.map → index.js.map} +0 -0
  117. /package/dist/browser/{browser/storage.d.ts → storage.d.ts} +0 -0
  118. /package/dist/browser/{browser/wasm.d.ts → wasm.d.ts} +0 -0
  119. /package/dist/browser/{browser/wasm.d.ts.map → wasm.d.ts.map} +0 -0
  120. /package/dist/browser/{browser/wasm.js.map → wasm.js.map} +0 -0
  121. /package/dist/node/{node/formats → formats}/index.d.ts +0 -0
  122. /package/dist/node/{node/formats → formats}/index.d.ts.map +0 -0
  123. /package/dist/node/{node/formats → formats}/index.js +0 -0
  124. /package/dist/node/{node/formats → formats}/index.js.map +0 -0
  125. /package/dist/node/{node/index.d.ts.map → index.d.ts.map} +0 -0
  126. /package/dist/node/{node/index.js.map → index.js.map} +0 -0
  127. /package/dist/node/{node/mime.d.ts → mime.d.ts} +0 -0
  128. /package/dist/node/{node/mime.d.ts.map → mime.d.ts.map} +0 -0
  129. /package/dist/node/{node/mime.js → mime.js} +0 -0
  130. /package/dist/node/{node/mime.js.map → mime.js.map} +0 -0
  131. /package/dist/node/{node/normalizers → normalizers}/binary.d.ts.map +0 -0
  132. /package/dist/node/{node/normalizers → normalizers}/dict.d.ts.map +0 -0
  133. /package/dist/node/{node/normalizers → normalizers}/dict.js +0 -0
  134. /package/dist/node/{node/normalizers → normalizers}/dict.js.map +0 -0
  135. /package/dist/node/{node/normalizers → normalizers}/files.d.ts.map +0 -0
  136. /package/dist/node/{node/normalizers → normalizers}/files.js.map +0 -0
  137. /package/dist/node/{node/normalizers → normalizers}/image.d.ts.map +0 -0
  138. /package/dist/node/{node/normalizers → normalizers}/index.d.ts +0 -0
  139. /package/dist/node/{node/normalizers → normalizers}/index.d.ts.map +0 -0
  140. /package/dist/node/{node/normalizers → normalizers}/index.js +0 -0
  141. /package/dist/node/{node/normalizers → normalizers}/index.js.map +0 -0
  142. /package/dist/node/{node/normalizers → normalizers}/pdf.d.ts.map +0 -0
  143. /package/dist/node/{node/normalizers → normalizers}/text.d.ts.map +0 -0
  144. /package/dist/node/{node/normalizers → normalizers}/video.d.ts.map +0 -0
  145. /package/dist/node/{node/wasm.d.ts → wasm.d.ts} +0 -0
  146. /package/dist/node/{node/wasm.d.ts.map → wasm.d.ts.map} +0 -0
  147. /package/dist/node/{node/wasm.js.map → wasm.js.map} +0 -0
@@ -4,9 +4,9 @@ exports.normalizeVideo = exports.normalizeImage = exports.normalizeBinary = expo
4
4
  exports.LeRobotEpisodes = exports.LeRobotFrames = exports.LeRobotTasks = exports.CocoPanoptic = exports.Coco = exports.normalizeHuggingface = exports.normalizeFiles = exports.normalizePdf = void 0;
5
5
  var client_1 = require("./client");
6
6
  Object.defineProperty(exports, "ManagedClient", { enumerable: true, get: function () { return client_1.ManagedClient; } });
7
- var database_1 = require("../shared/database");
7
+ var database_1 = require("./shared/database");
8
8
  Object.defineProperty(exports, "ManagedQueryBuilder", { enumerable: true, get: function () { return database_1.ManagedQueryBuilder; } });
9
- var errors_1 = require("../shared/errors");
9
+ var errors_1 = require("./shared/errors");
10
10
  Object.defineProperty(exports, "ManagedServiceError", { enumerable: true, get: function () { return errors_1.ManagedServiceError; } });
11
11
  Object.defineProperty(exports, "AuthError", { enumerable: true, get: function () { return errors_1.AuthError; } });
12
12
  Object.defineProperty(exports, "TokenError", { enumerable: true, get: function () { return errors_1.TokenError; } });
@@ -14,22 +14,22 @@ Object.defineProperty(exports, "CredentialsError", { enumerable: true, get: func
14
14
  Object.defineProperty(exports, "IngestError", { enumerable: true, get: function () { return errors_1.IngestError; } });
15
15
  Object.defineProperty(exports, "TableError", { enumerable: true, get: function () { return errors_1.TableError; } });
16
16
  Object.defineProperty(exports, "WorkspaceError", { enumerable: true, get: function () { return errors_1.WorkspaceError; } });
17
- var token_1 = require("../shared/token");
17
+ var token_1 = require("./shared/token");
18
18
  Object.defineProperty(exports, "extractOrgId", { enumerable: true, get: function () { return token_1.extractOrgId; } });
19
19
  Object.defineProperty(exports, "tryExtractOrgId", { enumerable: true, get: function () { return token_1.tryExtractOrgId; } });
20
20
  Object.defineProperty(exports, "fetchOrgIdFromApi", { enumerable: true, get: function () { return token_1.fetchOrgIdFromApi; } });
21
- var api_1 = require("../shared/api");
21
+ var api_1 = require("./shared/api");
22
22
  Object.defineProperty(exports, "apiRequest", { enumerable: true, get: function () { return api_1.apiRequest; } });
23
- var credentials_1 = require("../shared/credentials");
23
+ var credentials_1 = require("./shared/credentials");
24
24
  Object.defineProperty(exports, "getStorageCreds", { enumerable: true, get: function () { return credentials_1.getStorageCreds; } });
25
25
  Object.defineProperty(exports, "applyStorageCreds", { enumerable: true, get: function () { return credentials_1.applyStorageCreds; } });
26
- var schema_1 = require("../shared/schema");
26
+ var schema_1 = require("./shared/schema");
27
27
  Object.defineProperty(exports, "SCHEMA_TO_PG", { enumerable: true, get: function () { return schema_1.SCHEMA_TO_PG; } });
28
28
  Object.defineProperty(exports, "SCHEMA_TO_DEEPLAKE", { enumerable: true, get: function () { return schema_1.SCHEMA_TO_DEEPLAKE; } });
29
29
  Object.defineProperty(exports, "inferType", { enumerable: true, get: function () { return schema_1.inferType; } });
30
30
  Object.defineProperty(exports, "buildPgSchema", { enumerable: true, get: function () { return schema_1.buildPgSchema; } });
31
31
  Object.defineProperty(exports, "ensureColumns", { enumerable: true, get: function () { return schema_1.ensureColumns; } });
32
- var dlref_1 = require("../shared/dlref");
32
+ var dlref_1 = require("./shared/dlref");
33
33
  Object.defineProperty(exports, "isDataRef", { enumerable: true, get: function () { return dlref_1.isDataRef; } });
34
34
  Object.defineProperty(exports, "parseDataRef", { enumerable: true, get: function () { return dlref_1.parseDataRef; } });
35
35
  var mime_1 = require("./mime");
@@ -38,7 +38,7 @@ var wasm_1 = require("./wasm");
38
38
  Object.defineProperty(exports, "initializeWasm", { enumerable: true, get: function () { return wasm_1.initializeWasm; } });
39
39
  Object.defineProperty(exports, "deinitializeWasm", { enumerable: true, get: function () { return wasm_1.deinitializeWasm; } });
40
40
  Object.defineProperty(exports, "BUNDLED_WASM_PATH", { enumerable: true, get: function () { return wasm_1.BUNDLED_WASM_PATH; } });
41
- var wasm_common_1 = require("../shared/wasm-common");
41
+ var wasm_common_1 = require("./shared/wasm-common");
42
42
  Object.defineProperty(exports, "deeplakeOpen", { enumerable: true, get: function () { return wasm_common_1.deeplakeOpen; } });
43
43
  Object.defineProperty(exports, "deeplakeCreate", { enumerable: true, get: function () { return wasm_common_1.deeplakeCreate; } });
44
44
  Object.defineProperty(exports, "deeplakeExists", { enumerable: true, get: function () { return wasm_common_1.deeplakeExists; } });
@@ -0,0 +1,3 @@
1
+ import type { Batch } from '.././shared/types';
2
+ export declare function normalizeBinary(filePath: string): Batch;
3
+ //# sourceMappingURL=binary.d.ts.map
@@ -42,7 +42,7 @@ function normalizeBinary(filePath) {
42
42
  const data = fs.readFileSync(filePath);
43
43
  const filename = path.basename(filePath);
44
44
  return {
45
- id: [`${fileId}_0`],
45
+ chunk_id: [`${fileId}_0`],
46
46
  file_id: [fileId],
47
47
  data: [data],
48
48
  filename: [filename],
@@ -1 +1 @@
1
- {"version":3,"file":"binary.js","sourceRoot":"","sources":["../../../../src/node/normalizers/binary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,0CAWC;AAhBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,eAAe,CAAC,QAAgB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACnB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,IAAI,EAAE,CAAC,IAAI,CAAC;QACZ,QAAQ,EAAE,CAAC,QAAQ,CAAC;KACrB,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"binary.js","sourceRoot":"","sources":["../../../../src/node/normalizers/binary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,0CAWC;AAhBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,eAAe,CAAC,QAAgB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACzB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,IAAI,EAAE,CAAC,IAAI,CAAC;QACZ,QAAQ,EAAE,CAAC,QAAQ,CAAC;KACrB,CAAC;AACJ,CAAC"}
@@ -1,3 +1,3 @@
1
- import type { Batch } from '../../shared/types';
1
+ import type { Batch } from '.././shared/types';
2
2
  export declare function normalizeDict(data: Record<string, unknown[]>, batchSize?: number): Generator<Batch>;
3
3
  //# sourceMappingURL=dict.d.ts.map
@@ -1,3 +1,3 @@
1
- import type { Batch } from '../../shared/types';
1
+ import type { Batch } from '.././shared/types';
2
2
  export declare function normalizeFiles(files: string[], chunkSize: number, chunkOverlap: number, concurrency?: number): AsyncGenerator<Batch>;
3
3
  //# sourceMappingURL=files.d.ts.map
@@ -36,7 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.normalizeFiles = normalizeFiles;
37
37
  const path = __importStar(require("path"));
38
38
  const fs = __importStar(require("fs"));
39
- const errors_1 = require("../../shared/errors");
39
+ const errors_1 = require(".././shared/errors");
40
40
  const image_1 = require("./image");
41
41
  const text_1 = require("./text");
42
42
  const binary_1 = require("./binary");
@@ -0,0 +1,23 @@
1
+ /**
2
+ * HuggingFace dataset normalizer for Node.js.
3
+ *
4
+ * Streams a HuggingFace dataset via the `@huggingface/hub` or `datasets`
5
+ * package and yields column-oriented batches.
6
+ *
7
+ * This is a dynamic-import normalizer — it will throw a helpful error
8
+ * if no HuggingFace library is available.
9
+ */
10
+ import type { Batch } from '.././shared/types';
11
+ /**
12
+ * Normalize a HuggingFace dataset into column-oriented batches.
13
+ *
14
+ * Attempts to load the dataset using the `datasets` npm package first,
15
+ * falling back to `@huggingface/hub`. If neither is available, throws
16
+ * with installation instructions.
17
+ *
18
+ * @param datasetName - HuggingFace dataset identifier (e.g. "cifar10").
19
+ * @param batchSize - Rows per batch (default 100).
20
+ * @param split - Dataset split (default "train").
21
+ */
22
+ export declare function normalizeHuggingface(datasetName: string, batchSize?: number, split?: string): AsyncGenerator<Batch>;
23
+ //# sourceMappingURL=huggingface.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"huggingface.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/huggingface.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,wBAAuB,oBAAoB,CACzC,WAAW,EAAE,MAAM,EACnB,SAAS,GAAE,MAAY,EACvB,KAAK,GAAE,MAAgB,GACtB,cAAc,CAAC,KAAK,CAAC,CAgDvB"}
@@ -1,4 +1,13 @@
1
1
  "use strict";
2
+ /**
3
+ * HuggingFace dataset normalizer for Node.js.
4
+ *
5
+ * Streams a HuggingFace dataset via the `@huggingface/hub` or `datasets`
6
+ * package and yields column-oriented batches.
7
+ *
8
+ * This is a dynamic-import normalizer — it will throw a helpful error
9
+ * if no HuggingFace library is available.
10
+ */
2
11
  var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
12
  if (k2 === undefined) k2 = k;
4
13
  var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -34,10 +43,23 @@ var __importStar = (this && this.__importStar) || (function () {
34
43
  })();
35
44
  Object.defineProperty(exports, "__esModule", { value: true });
36
45
  exports.normalizeHuggingface = normalizeHuggingface;
46
+ /**
47
+ * Normalize a HuggingFace dataset into column-oriented batches.
48
+ *
49
+ * Attempts to load the dataset using the `datasets` npm package first,
50
+ * falling back to `@huggingface/hub`. If neither is available, throws
51
+ * with installation instructions.
52
+ *
53
+ * @param datasetName - HuggingFace dataset identifier (e.g. "cifar10").
54
+ * @param batchSize - Rows per batch (default 100).
55
+ * @param split - Dataset split (default "train").
56
+ */
37
57
  async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'train') {
58
+ // Try loading via @huggingface/hub (most common in Node.js)
38
59
  let rows;
39
60
  try {
40
61
  const hub = require('@huggingface/hub');
62
+ // @huggingface/hub provides a parquet-based streaming API
41
63
  if (typeof hub.listDatasetFiles === 'function') {
42
64
  rows = streamViaHub(hub, datasetName, split);
43
65
  }
@@ -46,6 +68,7 @@ async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'trai
46
68
  }
47
69
  }
48
70
  catch {
71
+ // Fallback: try to use a Python subprocess
49
72
  try {
50
73
  rows = streamViaPython(datasetName, split);
51
74
  }
@@ -59,6 +82,7 @@ async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'trai
59
82
  for await (const item of rows) {
60
83
  const row = {};
61
84
  for (const [col, val] of Object.entries(item)) {
85
+ // Convert Buffer/Uint8Array values to Buffer for consistency
62
86
  if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
63
87
  row[col] = Buffer.from(val);
64
88
  }
@@ -77,6 +101,9 @@ async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'trai
77
101
  }
78
102
  }
79
103
  async function* streamViaHub(hub, datasetName, split) {
104
+ // Use the hub's dataset viewer API for streaming
105
+ const { createRepo, downloadFile } = hub;
106
+ // Try the datasets server API (works for most public datasets)
80
107
  const baseUrl = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(datasetName)}&config=default&split=${encodeURIComponent(split)}`;
81
108
  let offset = 0;
82
109
  const length = 100;
@@ -125,6 +152,7 @@ for item in ds:
125
152
  if (!trimmed)
126
153
  continue;
127
154
  const row = JSON.parse(trimmed);
155
+ // Decode base64-encoded bytes back to Buffer
128
156
  for (const [key, val] of Object.entries(row)) {
129
157
  if (val &&
130
158
  typeof val === 'object' &&
@@ -134,6 +162,7 @@ for item in ds:
134
162
  }
135
163
  yield row;
136
164
  }
165
+ // Wait for process to exit
137
166
  await new Promise((resolve, reject) => {
138
167
  proc.on('close', (code) => {
139
168
  if (code !== 0) {
@@ -0,0 +1 @@
1
+ {"version":3,"file":"huggingface.js","sourceRoot":"","sources":["../../../../src/node/normalizers/huggingface.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAeH,oDAoDC;AA/DD;;;;;;;;;;GAUG;AACI,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,WAAmB,EACnB,YAAoB,GAAG,EACvB,QAAgB,OAAO;IAEvB,4DAA4D;IAC5D,IAAI,IAA4C,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,CAAC,kBAAkB,CAAQ,CAAC;QAC/C,0DAA0D;QAC1D,IAAI,OAAO,GAAG,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;YAC/C,IAAI,GAAG,YAAY,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2CAA2C;QAC3C,IAAI,CAAC;YACH,IAAI,GAAG,eAAe,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,6DAA6D;gBAC7D,kCAAkC;gBAClC,0DAA0D,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAA8B,EAAE,CAAC;IAE5C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,6DAA6D;YAC7D,IAAI,GAAG,YAAY,UAAU,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YACjB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;YACzB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,YAAY,CAC1B,GAAQ,EACR,WAAmB,EACnB,KAAa;IAEb,iDAAiD;IACjD,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC;IAEzC,+DAA+D;IAC/D,MAAM,OAAO,GAAG,uDAAuD,kBAAkB,CAAC,WAAW,CAAC,yBAAyB,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IAE3J,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,GAAG,CAAC;IAEnB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,GAAG,OAAO,WAAW,MAAM,WAAW,MAAM,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM;QAEpB,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAQ,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAa,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAEtC,KAAK,MAAM,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9B,MAAM,UAAU,CAAC,GAA8B,CAAC;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM;YAAE,MAAM;QAChC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAC7B,WAAmB,EACnB,KAAa;IAEb,MAAM,EAAE,KAAK,EAAE,GAAG,wDAAa,eAAe,GAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,wDAAa,UAAU,GAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG;;;qBAGE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,aAAa,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;;;;;;;;;;;;CAY3F,CAAC;IAEA,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE;QAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;KAChC,CAAC,CAAC;IAEH,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAO,EAAE,CAAC,CAAC;IAE7D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;QAE3D,6CAA6C;QAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IACE,GAAG;gBACH,OAAO,GAAG,KAAK,QAAQ;gBACvB,WAAW,IAAK,GAA+B,EAC/C,CAAC;gBACD,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CACnB,GAA6B,CAAC,SAAS,EACxC,QAAQ,CACT,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,2BAA2B;IAC3B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,IAA+B;IAClD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -0,0 +1,3 @@
1
+ import type { Batch } from '.././shared/types';
2
+ export declare function normalizeImage(filePath: string): Batch;
3
+ //# sourceMappingURL=image.d.ts.map
@@ -42,7 +42,7 @@ function normalizeImage(filePath) {
42
42
  const data = fs.readFileSync(filePath);
43
43
  const filename = path.basename(filePath);
44
44
  return {
45
- id: [`${fileId}_0`],
45
+ chunk_id: [`${fileId}_0`],
46
46
  file_id: [fileId],
47
47
  image: [data],
48
48
  filename: [filename],
@@ -1 +1 @@
1
- {"version":3,"file":"image.js","sourceRoot":"","sources":["../../../../src/node/normalizers/image.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,wCAYC;AAjBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,cAAc,CAAC,QAAgB;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACnB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;KAC7B,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"image.js","sourceRoot":"","sources":["../../../../src/node/normalizers/image.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,wCAYC;AAjBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,cAAc,CAAC,QAAgB;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACzB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;KAC7B,CAAC;AACJ,CAAC"}
@@ -0,0 +1,3 @@
1
+ import type { Batch } from '.././shared/types';
2
+ export declare function normalizePdf(filePath: string): AsyncGenerator<Batch>;
3
+ //# sourceMappingURL=pdf.d.ts.map
@@ -47,7 +47,7 @@ async function* normalizePdf(filePath) {
47
47
  catch {
48
48
  // Fallback: store whole PDF as binary
49
49
  yield {
50
- id: [`${fileId}_0`],
50
+ chunk_id: [`${fileId}_0`],
51
51
  file_id: [fileId],
52
52
  page_index: [0],
53
53
  data: [fs.readFileSync(filePath)],
@@ -64,7 +64,7 @@ async function* normalizePdf(filePath) {
64
64
  .map((item) => item.str)
65
65
  .join(' ');
66
66
  yield {
67
- id: [`${fileId}_${pageIdx}`],
67
+ chunk_id: [`${fileId}_${pageIdx}`],
68
68
  file_id: [fileId],
69
69
  page_index: [pageIdx],
70
70
  text: [text || `PDF page ${pageIdx + 1}`],
@@ -1 +1 @@
1
- {"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../../src/node/normalizers/pdf.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,oCAsCC;AA3CD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAG1B,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,QAAgB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,IAAI,KAAU,CAAC;IACf,IAAI,CAAC;QACH,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;QACtC,MAAM;YACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACnB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,CAAC,CAAC;YACf,IAAI,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,EAAE,CAAC,QAAQ,QAAQ,EAAE,CAAC;SAC3B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC;IAEtD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK;aAC3B,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;aAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM;YACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,OAAO,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,OAAO,CAAC;YACrB,IAAI,EAAE,CAAC,IAAI,IAAI,YAAY,OAAO,GAAG,CAAC,EAAE,CAAC;SAC1C,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,OAAO,EAAE,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../../src/node/normalizers/pdf.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,oCAsCC;AA3CD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAG1B,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,QAAgB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,IAAI,KAAU,CAAC;IACf,IAAI,CAAC;QACH,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;QACtC,MAAM;YACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACzB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,CAAC,CAAC;YACf,IAAI,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,EAAE,CAAC,QAAQ,QAAQ,EAAE,CAAC;SAC3B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC;IAEtD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK;aAC3B,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;aAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM;YACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,OAAO,EAAE,CAAC;YAClC,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,OAAO,CAAC;YACrB,IAAI,EAAE,CAAC,IAAI,IAAI,YAAY,OAAO,GAAG,CAAC,EAAE,CAAC;SAC1C,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,OAAO,EAAE,CAAC;AAChB,CAAC"}
@@ -1,4 +1,4 @@
1
- import type { Batch } from '../../shared/types';
1
+ import type { Batch } from '.././shared/types';
2
2
  export declare function chunkText(text: string, chunkSize: number, overlap: number): string[];
3
3
  export declare function normalizeText(filePath: string, chunkSize: number, chunkOverlap: number): Batch;
4
4
  //# sourceMappingURL=text.d.ts.map
@@ -74,7 +74,7 @@ function normalizeText(filePath, chunkSize, chunkOverlap) {
74
74
  const chunks = chunkText(content, chunkSize, chunkOverlap);
75
75
  const name = filePath.split('/').pop() ?? filePath;
76
76
  return {
77
- id: chunks.map((_, i) => `${fileId}_${i}`),
77
+ chunk_id: chunks.map((_, i) => `${fileId}_${i}`),
78
78
  file_id: Array(chunks.length).fill(fileId),
79
79
  chunk_index: chunks.map((_, i) => i),
80
80
  text: chunks.length > 0 ? chunks : [`Text: ${name}`],
@@ -1 +1 @@
1
- {"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/node/normalizers/text.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,8BAwCC;AAED,sCAgBC;AA9DD,uCAAyB;AACzB,+CAAiC;AAGjC,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,OAAe;IAEf,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,KAAK,GAAG,SAAS,CAAC;QAE5B,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM;QACR,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7D,IAAI,SAAS,GAAG,GAAG,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;YAC5C,IAAI,GAAG,IAAI,WAAW,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;gBACtC,SAAS,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;QACrC,KAAK,GAAG,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,QAAgB,EAChB,SAAiB,EACjB,YAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IAEnD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/node/normalizers/text.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,8BAwCC;AAED,sCAgBC;AA9DD,uCAAyB;AACzB,+CAAiC;AAGjC,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,OAAe;IAEf,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,KAAK,GAAG,SAAS,CAAC;QAE5B,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM;QACR,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7D,IAAI,SAAS,GAAG,GAAG,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;YAC5C,IAAI,GAAG,IAAI,WAAW,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;gBACtC,SAAS,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;QACrC,KAAK,GAAG,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,QAAgB,EAChB,SAAiB,EACjB,YAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IAEnD,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC;QAChD,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}
@@ -1,3 +1,3 @@
1
- import type { Batch } from '../../shared/types';
1
+ import type { Batch } from '.././shared/types';
2
2
  export declare function normalizeVideo(filePath: string, chunkInterval?: number): AsyncGenerator<Batch>;
3
3
  //# sourceMappingURL=video.d.ts.map
@@ -61,7 +61,7 @@ async function* normalizeVideo(filePath, chunkInterval = 10) {
61
61
  const duration = await probeDuration(filePath);
62
62
  if (duration <= 0) {
63
63
  yield {
64
- id: [`${fileId}_0`],
64
+ chunk_id: [`${fileId}_0`],
65
65
  file_id: [fileId],
66
66
  chunk_index: [0],
67
67
  start_time: [0.0],
@@ -101,7 +101,7 @@ async function* normalizeVideo(filePath, chunkInterval = 10) {
101
101
  }
102
102
  const videoData = fs.existsSync(segPath) ? fs.readFileSync(segPath) : null;
103
103
  yield {
104
- id: [`${fileId}_${idx}`],
104
+ chunk_id: [`${fileId}_${idx}`],
105
105
  file_id: [fileId],
106
106
  chunk_index: [idx],
107
107
  start_time: [Math.round(start * 1000) / 1000],
@@ -1 +1 @@
1
- {"version":3,"file":"video.js","sourceRoot":"","sources":["../../../../src/node/normalizers/video.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,wCA2EC;AAnGD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AACjC,iDAAyC;AACzC,+BAAiC;AAGjC,MAAM,aAAa,GAAG,IAAA,gBAAS,EAAC,wBAAQ,CAAC,CAAC;AAE1C,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE;YAChD,IAAI,EAAE,OAAO;YACb,eAAe,EAAE,MAAM;YACvB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,IAAI,GAAG,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,QAAgB,EAChB,gBAAwB,EAAE;IAE1B,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;IAE/C,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM;YACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACnB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,WAAW,EAAE,CAAC,CAAC,CAAC;YAChB,UAAU,EAAE,CAAC,GAAG,CAAC;YACjB,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,UAAU,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvC,SAAS,EAAE,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;SAC7B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,wDAAa,IAAI,GAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;IAE5B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,GAAG,GAAG,aAAa,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,KAAK,GAAG,WAAW,CAAC;QAEhC,MAAM,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAC9F,EAAE,OAAO,EAAE,OAAO,EAAE,CACrB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EACrG,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAC;gBACF,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7B,SAAS,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YAED,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAE3E,MAAM;gBACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBAC7C,QAAQ,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBACzC,UAAU,EAAE,CAAC,SAAS,CAAC;gBACvB,SAAS,EAAE,CAAC,SAAS,CAAC;gBACtB,IAAI,EAAE,CAAC,iBAAiB,GAAG,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;aAC1E,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"video.js","sourceRoot":"","sources":["../../../../src/node/normalizers/video.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,wCA2EC;AAnGD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AACjC,iDAAyC;AACzC,+BAAiC;AAGjC,MAAM,aAAa,GAAG,IAAA,gBAAS,EAAC,wBAAQ,CAAC,CAAC;AAE1C,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE;YAChD,IAAI,EAAE,OAAO;YACb,eAAe,EAAE,MAAM;YACvB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,IAAI,GAAG,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,QAAgB,EAChB,gBAAwB,EAAE;IAE1B,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;IAE/C,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM;YACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACzB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,WAAW,EAAE,CAAC,CAAC,CAAC;YAChB,UAAU,EAAE,CAAC,GAAG,CAAC;YACjB,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,UAAU,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvC,SAAS,EAAE,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;SAC7B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,wDAAa,IAAI,GAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;IAE5B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,GAAG,GAAG,aAAa,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,KAAK,GAAG,WAAW,CAAC;QAEhC,MAAM,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAC9F,EAAE,OAAO,EAAE,OAAO,EAAE,CACrB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EACrG,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAC;gBACF,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7B,SAAS,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YAED,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAE3E,MAAM;gBACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC;gBAC9B,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBAC7C,QAAQ,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBACzC,UAAU,EAAE,CAAC,SAAS,CAAC;gBACvB,SAAS,EAAE,CAAC,SAAS,CAAC;gBACtB,IAAI,EAAE,CAAC,iBAAiB,GAAG,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;aAC1E,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE,MAAoB,GACzB,OAAO,CAAC,YAAY,CAAC,CAYvB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAQ3D"}
1
+ {"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE,MAAoB,GACzB,OAAO,CAAC,YAAY,CAAC,CAYvB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAM3D"}
@@ -17,12 +17,10 @@ async function getStorageCreds(baseUrl, token, orgId, workspaceId, mode = 'readw
17
17
  };
18
18
  }
19
19
  function applyStorageCreds(creds) {
20
- if (typeof process !== 'undefined' && process.env) {
21
- if (creds.aws_access_key_id) {
22
- process.env.AWS_ACCESS_KEY_ID = creds.aws_access_key_id;
23
- process.env.AWS_SECRET_ACCESS_KEY = creds.aws_secret_access_key;
24
- process.env.AWS_SESSION_TOKEN = creds.aws_session_token ?? '';
25
- }
20
+ if (creds.aws_access_key_id) {
21
+ process.env.AWS_ACCESS_KEY_ID = creds.aws_access_key_id;
22
+ process.env.AWS_SECRET_ACCESS_KEY = creds.aws_secret_access_key;
23
+ process.env.AWS_SESSION_TOKEN = creds.aws_session_token ?? '';
26
24
  }
27
25
  }
28
26
  //# sourceMappingURL=credentials.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":";;AAGA,0CAkBC;AAED,8CAQC;AA/BD,+BAAmC;AAG5B,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,KAAa,EACb,KAAa,EACb,WAAmB,EACnB,OAAe,WAAW;IAE1B,MAAM,IAAI,GAAG,MAAM,IAAA,gBAAU,EAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE;QACnD,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,eAAe,WAAW,uBAAuB;QACvD,IAAI,EAAE,EAAE,IAAI,EAAE;KACf,CAAC,CAAC;IACH,MAAM,KAAK,GAAI,IAAI,CAAC,KAAgC,IAAI,IAAI,CAAC;IAC7D,OAAO;QACL,iBAAiB,EAAE,KAAK,CAAC,iBAA2B;QACpD,qBAAqB,EAAE,KAAK,CAAC,qBAA+B;QAC5D,iBAAiB,EAAE,KAAK,CAAC,iBAAuC;KACjE,CAAC;AACJ,CAAC;AAED,SAAgB,iBAAiB,CAAC,KAAmB;IACnD,IAAI,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,KAAK,CAAC,qBAAqB,CAAC;YAChE,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC;QAChE,CAAC;IACH,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":";;AAGA,0CAkBC;AAED,8CAMC;AA7BD,+BAAmC;AAG5B,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,KAAa,EACb,KAAa,EACb,WAAmB,EACnB,OAAe,WAAW;IAE1B,MAAM,IAAI,GAAG,MAAM,IAAA,gBAAU,EAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE;QACnD,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,eAAe,WAAW,uBAAuB;QACvD,IAAI,EAAE,EAAE,IAAI,EAAE;KACf,CAAC,CAAC;IACH,MAAM,KAAK,GAAI,IAAI,CAAC,KAAgC,IAAI,IAAI,CAAC;IAC7D,OAAO;QACL,iBAAiB,EAAE,KAAK,CAAC,iBAA2B;QACpD,qBAAqB,EAAE,KAAK,CAAC,qBAA+B;QAC5D,iBAAiB,EAAE,KAAK,CAAC,iBAAuC;KACjE,CAAC;AACJ,CAAC;AAED,SAAgB,iBAAiB,CAAC,KAAmB;IACnD,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,KAAK,CAAC,qBAAqB,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAChE,CAAC;AACH,CAAC"}
@@ -1,6 +1,8 @@
1
1
  import type { QueryRow } from './types';
2
2
  export interface QueryExecutor {
3
- query(sql: string, params?: unknown[]): Promise<QueryRow[]>;
3
+ query(sql: string, params?: unknown[], options?: {
4
+ timeoutMs?: number;
5
+ }): Promise<QueryRow[]>;
4
6
  }
5
7
  export declare class ManagedQueryBuilder {
6
8
  private _executor;
@@ -1 +1 @@
1
- {"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/shared/database.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CAC7D;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,WAAW,CAAkD;IACrE,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,UAAU,CAAuB;gBAE7B,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM;IAKlD,MAAM,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI;IAKlC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI;IAKpD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK7B,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKtB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKvB,QAAQ,IAAI;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;KAAE;IA+B/C,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKpC,QAAQ,IAAI,MAAM;CAInB"}
1
+ {"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/shared/database.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CAC/F;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,WAAW,CAAkD;IACrE,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,UAAU,CAAuB;gBAE7B,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM;IAKlD,MAAM,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI;IAKlC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI;IAKpD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK7B,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKtB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKvB,QAAQ,IAAI;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;KAAE;IA+B/C,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKpC,QAAQ,IAAI,MAAM;CAInB"}
@@ -11,6 +11,9 @@ export interface DataRef {
11
11
  columnId: number;
12
12
  rowId: bigint;
13
13
  }
14
+ /**
15
+ * Check if a value is a DLREF data reference.
16
+ */
14
17
  export declare function isDataRef(value: unknown): boolean;
15
18
  /**
16
19
  * Parse a DLREF data reference string into its components.
@@ -1 +1 @@
1
- {"version":3,"file":"dlref.d.ts","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAID,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CASjD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,OAAO,CAuBhE"}
1
+ {"version":3,"file":"dlref.d.ts","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAID;;GAEG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CASjD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,OAAO,CAuBhE"}
@@ -11,6 +11,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
11
11
  exports.isDataRef = isDataRef;
12
12
  exports.parseDataRef = parseDataRef;
13
13
  const DLREF_PREFIX = 'DLREF:';
14
+ /**
15
+ * Check if a value is a DLREF data reference.
16
+ */
14
17
  function isDataRef(value) {
15
18
  if (typeof value === 'string') {
16
19
  return value.startsWith(DLREF_PREFIX);
@@ -1 +1 @@
1
- {"version":3,"file":"dlref.js","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AAUH,8BASC;AAQD,oCAuBC;AA1CD,MAAM,YAAY,GAAG,QAAQ,CAAC;AAE9B,SAAgB,SAAS,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9D,OAAO,MAAM,KAAK,YAAY,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,SAAgB,YAAY,CAAC,KAA0B;IACrD,IAAI,GAAW,CAAC;IAChB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,GAAG,GAAG,KAAK,CAAC;IACd,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,+CAA+C,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACjB,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAChC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"dlref.js","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AAaH,8BASC;AAQD,oCAuBC;AA7CD,MAAM,YAAY,GAAG,QAAQ,CAAC;AAE9B;;GAEG;AACH,SAAgB,SAAS,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9D,OAAO,MAAM,KAAK,YAAY,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,SAAgB,YAAY,CAAC,KAA0B;IACrD,IAAI,GAAW,CAAC;IAChB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,GAAG,GAAG,KAAK,CAAC;IACd,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,+CAA+C,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACjB,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAChC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,CAAC;AACJ,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAExD,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqB/C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqBrD,CAAC;AAEF,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAqBhD;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,KAAK,EACZ,MAAM,CAAC,EAAE,SAAS,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CA2BxB;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,GAAG,EACZ,KAAK,EAAE,KAAK,EACZ,MAAM,CAAC,EAAE,SAAS,GACjB,IAAI,CAiDN"}
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAExD,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqB/C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqBrD,CAAC;AAEF,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAmBhD;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,KAAK,EACZ,MAAM,CAAC,EAAE,SAAS,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAwCxB;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,GAAG,EACZ,KAAK,EAAE,KAAK,EACZ,MAAM,CAAC,EAAE,SAAS,GACjB,IAAI,CAiDN"}
@@ -14,17 +14,17 @@ exports.SCHEMA_TO_PG = {
14
14
  FLOAT64: 'DOUBLE PRECISION',
15
15
  BOOL: 'BOOLEAN',
16
16
  BINARY: 'BYTEA',
17
- IMAGE: 'BYTEA',
17
+ IMAGE: 'IMAGE',
18
18
  VIDEO: 'VIDEO',
19
19
  EMBEDDING: 'float4[]',
20
- SEGMENT_MASK: 'BYTEA',
21
- BINARY_MASK: 'BYTEA',
22
- BOUNDING_BOX: 'float4[]',
23
- CLASS_LABEL: 'INTEGER',
24
- POLYGON: 'BYTEA',
25
- POINT: 'float4[]',
26
- MESH: 'BYTEA',
27
- MEDICAL: 'BYTEA',
20
+ SEGMENT_MASK: 'SEGMENT_MASK',
21
+ BINARY_MASK: 'BINARY_MASK',
22
+ BOUNDING_BOX: 'BOUNDING_BOX',
23
+ CLASS_LABEL: 'CLASS_LABEL',
24
+ POLYGON: 'DEEPLAKE_POLYGON',
25
+ POINT: 'DEEPLAKE_POINT',
26
+ MESH: 'MESH',
27
+ MEDICAL: 'MEDICAL',
28
28
  };
29
29
  exports.SCHEMA_TO_DEEPLAKE = {
30
30
  TEXT: 'text',
@@ -36,17 +36,17 @@ exports.SCHEMA_TO_DEEPLAKE = {
36
36
  FLOAT64: 'float64',
37
37
  BOOL: 'bool',
38
38
  BINARY: 'bytes',
39
- IMAGE: 'image',
39
+ IMAGE: 'bytes',
40
40
  VIDEO: 'bytes',
41
41
  EMBEDDING: 'float32',
42
- SEGMENT_MASK: 'segment_mask',
43
- BINARY_MASK: 'binary_mask',
44
- BOUNDING_BOX: 'bounding_box',
45
- CLASS_LABEL: 'class_label',
46
- POLYGON: 'polygon',
47
- POINT: 'point',
48
- MESH: 'mesh',
49
- MEDICAL: 'medical',
42
+ SEGMENT_MASK: 'bytes',
43
+ BINARY_MASK: 'bytes',
44
+ BOUNDING_BOX: 'float32',
45
+ CLASS_LABEL: 'int32',
46
+ POLYGON: 'bytes',
47
+ POINT: 'float32',
48
+ MESH: 'bytes',
49
+ MEDICAL: 'bytes',
50
50
  };
51
51
  function inferType(value) {
52
52
  if (typeof value === 'boolean')
@@ -58,9 +58,8 @@ function inferType(value) {
58
58
  return 'TEXT';
59
59
  if ((typeof Buffer !== 'undefined' && Buffer.isBuffer(value)) ||
60
60
  value instanceof Uint8Array ||
61
- value instanceof ArrayBuffer) {
61
+ value instanceof ArrayBuffer)
62
62
  return 'BINARY';
63
- }
64
63
  if (Array.isArray(value) &&
65
64
  value.length > 0 &&
66
65
  value.every((x) => typeof x === 'number')) {
@@ -91,6 +90,18 @@ function buildPgSchema(batch, schema, format) {
91
90
  }
92
91
  }
93
92
  }
93
+ // Add _thumbnail columns for each IMAGE column. Thumbnails are
94
+ // stored inline in the same table (128×128 JPEG) rather than in
95
+ // a separate thumbnails table. Use BYTEA (not IMAGE domain) since
96
+ // thumbnails are derived data, not primary images.
97
+ for (const [colName, pgType] of Object.entries({ ...pgSchema })) {
98
+ if (pgType.toUpperCase() === 'IMAGE') {
99
+ const thumbCol = `${colName}_thumbnail`;
100
+ if (!(thumbCol in pgSchema)) {
101
+ pgSchema[thumbCol] = 'IMAGE';
102
+ }
103
+ }
104
+ }
94
105
  return pgSchema;
95
106
  }
96
107
  function ensureColumns(dataset, batch, schema) {
@@ -1 +1 @@
1
- {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":";;;AAgDA,8BAqBC;AAED,sCA+BC;AAED,sCAqDC;AA3JY,QAAA,YAAY,GAA2B;IAClD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,QAAQ;IACb,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,kBAAkB;IACzB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,kBAAkB;IAC3B,IAAI,EAAE,SAAS;IACf,MAAM,EAAE,OAAO;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,OAAO;IACrB,WAAW,EAAE,OAAO;IACpB,YAAY,EAAE,UAAU;IACxB,WAAW,EAAE,SAAS;IACtB,OAAO,EAAE,OAAO;IAChB,KAAK,EAAE,UAAU;IACjB,IAAI,EAAE,OAAO;IACb,OAAO,EAAE,OAAO;CACjB,CAAC;AAEW,QAAA,kBAAkB,GAA2B;IACxD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,OAAO;IACZ,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,SAAS;IAClB,OAAO,EAAE,SAAS;IAClB,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,OAAO;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,SAAS;IACpB,YAAY,EAAE,cAAc;IAC5B,WAAW,EAAE,aAAa;IAC1B,YAAY,EAAE,cAAc;IAC5B,WAAW,EAAE,aAAa;IAC1B,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;CACnB,CAAC;AAEF,SAAgB,SAAS,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACvD,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7C,IACE,CAAC,OAAO,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzD,KAAK,YAAY,UAAU;QAC3B,KAAK,YAAY,WAAW,EAC5B,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IACE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACpB,KAAK,CAAC,MAAM,GAAG,CAAC;QAChB,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EACzC,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,KAAY,EACZ,MAAkB,EAClB,MAAe;IAEf,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAE5C,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,QAAQ,KAAK,MAAM;gBAAE,SAAS;YAClC,QAAQ,CAAC,OAAO,CAAC,GAAG,oBAAY,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/E,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,QAAQ,CAAC,OAAO,CAAC,GAAG,oBAAY,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;IACvD,CAAC;IAED,8EAA8E;IAC9E,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YACnE,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,QAAQ,CAAC,OAAO,CAAC,GAAG,UAAU,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAgB,aAAa,CAC3B,OAAY,EACZ,KAAY,EACZ,MAAkB;IAElB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,SAAS,GAAQ,IAAI,CAAC;IAC1B,IAAI,CAAC;QACH,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC;YAC/B,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBAC5C,uCAAuC;gBACvC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACxB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACvB,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,UAAU;wBAAE,GAAG,CAAC,MAAM,EAAE,CAAC;gBACrD,CAAC;gBACD,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU;oBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACvD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,KAAK,MAAM,CAAC,IAAI,IAAI;oBAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;YAAS,CAAC;QACT,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxD,IAAI,CAAC;gBAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QAEpC,IAAI,MAAc,CAAC;QACnB,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,QAAQ,KAAK,MAAM;gBAAE,SAAS;YAClC,MAAM,GAAG,0BAAkB,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC/E,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,GAAG,0BAAkB,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;QAClD,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAAE,MAAM,CAAC,CAAC;QACnE,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":";;;AAgDA,8BAmBC;AAED,sCA4CC;AAED,sCAqDC;AAtKY,QAAA,YAAY,GAA2B;IAClD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,QAAQ;IACb,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,kBAAkB;IACzB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,kBAAkB;IAC3B,IAAI,EAAE,SAAS;IACf,MAAM,EAAE,OAAO;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,cAAc;IAC5B,WAAW,EAAE,aAAa;IAC1B,YAAY,EAAE,cAAc;IAC5B,WAAW,EAAE,aAAa;IAC1B,OAAO,EAAE,kBAAkB;IAC3B,KAAK,EAAE,gBAAgB;IACvB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;CACnB,CAAC;AAEW,QAAA,kBAAkB,GAA2B;IACxD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,OAAO;IACZ,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,SAAS;IAClB,OAAO,EAAE,SAAS;IAClB,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,OAAO;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,SAAS;IACpB,YAAY,EAAE,OAAO;IACrB,WAAW,EAAE,OAAO;IACpB,YAAY,EAAE,SAAS;IACvB,WAAW,EAAE,OAAO;IACpB,OAAO,EAAE,OAAO;IAChB,KAAK,EAAE,SAAS;IAChB,IAAI,EAAE,OAAO;IACb,OAAO,EAAE,OAAO;CACjB,CAAC;AAEF,SAAgB,SAAS,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACvD,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7C,IACE,CAAC,OAAO,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzD,KAAK,YAAY,UAAU;QAC3B,KAAK,YAAY,WAAW;QAC5B,OAAO,QAAQ,CAAC;IAClB,IACE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACpB,KAAK,CAAC,MAAM,GAAG,CAAC;QAChB,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EACzC,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,KAAY,EACZ,MAAkB,EAClB,MAAe;IAEf,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAE5C,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,QAAQ,KAAK,MAAM;gBAAE,SAAS;YAClC,QAAQ,CAAC,OAAO,CAAC,GAAG,oBAAY,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/E,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,QAAQ,CAAC,OAAO,CAAC,GAAG,oBAAY,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;IACvD,CAAC;IAED,8EAA8E;IAC9E,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YACnE,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,QAAQ,CAAC,OAAO,CAAC,GAAG,UAAU,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,gEAAgE;IAChE,mEAAmE;IACnE,mDAAmD;IACnD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAC,GAAG,QAAQ,EAAC,CAAC,EAAE,CAAC;QAC9D,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,GAAG,OAAO,YAAY,CAAC;YACxC,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAgB,aAAa,CAC3B,OAAY,EACZ,KAAY,EACZ,MAAkB;IAElB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,SAAS,GAAQ,IAAI,CAAC;IAC1B,IAAI,CAAC;QACH,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC;YAC/B,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBAC5C,uCAAuC;gBACvC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACxB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACvB,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,UAAU;wBAAE,GAAG,CAAC,MAAM,EAAE,CAAC;gBACrD,CAAC;gBACD,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU;oBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACvD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,KAAK,MAAM,CAAC,IAAI,IAAI;oBAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;YAAS,CAAC;QACT,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxD,IAAI,CAAC;gBAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QAEpC,IAAI,MAAc,CAAC;QACnB,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,QAAQ,KAAK,MAAM;gBAAE,SAAS;YAClC,MAAM,GAAG,0BAAkB,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC/E,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,GAAG,0BAAkB,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;QAClD,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAAE,MAAM,CAAC,CAAC;QACnE,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":"AAmBA,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAyBlD;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM5D;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgBxB"}
1
+ {"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":"AAEA,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAyBlD;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM5D;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgBxB"}
@@ -4,29 +4,13 @@ exports.extractOrgId = extractOrgId;
4
4
  exports.tryExtractOrgId = tryExtractOrgId;
5
5
  exports.fetchOrgIdFromApi = fetchOrgIdFromApi;
6
6
  const errors_1 = require("./errors");
7
- function base64UrlDecode(str) {
8
- if (typeof Buffer !== 'undefined') {
9
- return Buffer.from(str, 'base64url').toString('utf-8');
10
- }
11
- // Browser fallback using atob + TextDecoder
12
- let base64 = str.replace(/-/g, '+').replace(/_/g, '/');
13
- while (base64.length % 4 !== 0) {
14
- base64 += '=';
15
- }
16
- const binary = atob(base64);
17
- const bytes = new Uint8Array(binary.length);
18
- for (let i = 0; i < binary.length; i++) {
19
- bytes[i] = binary.charCodeAt(i);
20
- }
21
- return new TextDecoder().decode(bytes);
22
- }
23
7
  function extractOrgId(token) {
24
8
  try {
25
9
  const parts = token.split('.');
26
10
  if (parts.length !== 3) {
27
11
  throw new errors_1.TokenError('Invalid token format: expected JWT with 3 parts');
28
12
  }
29
- const decoded = base64UrlDecode(parts[1]);
13
+ const decoded = Buffer.from(parts[1], 'base64url').toString('utf-8');
30
14
  const claims = JSON.parse(decoded);
31
15
  const orgId = claims.OrgID || claims.org_id;
32
16
  if (!orgId) {
@@ -1 +1 @@
1
- {"version":3,"file":"token.js","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":";;AAmBA,oCAyBC;AAED,0CAMC;AAED,8CAmBC;AAzED,qCAAsC;AAEtC,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzD,CAAC;IACD,4CAA4C;IAC5C,IAAI,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAC;IAChB,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzC,CAAC;AAED,SAAgB,YAAY,CAAC,KAAa;IACxC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,mBAAU,CAAC,iDAAiD,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC;QAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,mBAAU,CAAC,qCAAqC,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,mBAAU;YAAE,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,YAAY,WAAW,EAAE,CAAC;YAC7B,MAAM,IAAI,mBAAU,CAAC,mCAAmC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,IAAI,mBAAU,CAClB,wCAAwC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAC7E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAgB,eAAe,CAAC,KAAa;IAC3C,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,KAAK,EAAE;YACvC,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,KAAK,EAAE;gBAChC,cAAc,EAAE,kBAAkB;aACnC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA4B,CAAC;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,aAAqC,CAAC;QACxD,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"token.js","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":";;AAEA,oCAyBC;AAED,0CAMC;AAED,8CAmBC;AAxDD,qCAAsC;AAEtC,SAAgB,YAAY,CAAC,KAAa;IACxC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,mBAAU,CAAC,iDAAiD,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC;QAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,mBAAU,CAAC,qCAAqC,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,mBAAU;YAAE,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,YAAY,WAAW,EAAE,CAAC;YAC7B,MAAM,IAAI,mBAAU,CAAC,mCAAmC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,IAAI,mBAAU,CAClB,wCAAwC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAC7E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAgB,eAAe,CAAC,KAAa;IAC3C,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,KAAK,EAAE;YACvC,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,KAAK,EAAE;gBAChC,cAAc,EAAE,kBAAkB;aACnC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA4B,CAAC;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,aAAqC,CAAC;QACxD,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}