deeplake 0.3.16 → 0.3.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{browser/client.d.ts → client.d.ts} +2 -2
- package/dist/browser/{browser/client.js → client.js} +8 -8
- package/dist/browser/{browser/index.d.ts → index.d.ts} +12 -12
- package/dist/browser/{browser/index.js → index.js} +8 -8
- package/dist/browser/shared/credentials.d.ts.map +1 -1
- package/dist/browser/shared/credentials.js +4 -6
- package/dist/browser/shared/credentials.js.map +1 -1
- package/dist/browser/shared/database.d.ts +3 -1
- package/dist/browser/shared/database.d.ts.map +1 -1
- package/dist/browser/shared/dlref.d.ts +3 -0
- package/dist/browser/shared/dlref.d.ts.map +1 -1
- package/dist/browser/shared/dlref.js +3 -0
- package/dist/browser/shared/dlref.js.map +1 -1
- package/dist/browser/shared/schema.d.ts.map +1 -1
- package/dist/browser/shared/schema.js +31 -20
- package/dist/browser/shared/schema.js.map +1 -1
- package/dist/browser/shared/token.d.ts.map +1 -1
- package/dist/browser/shared/token.js +1 -17
- package/dist/browser/shared/token.js.map +1 -1
- package/dist/browser/{browser/storage.d.ts.map → storage.d.ts.map} +1 -1
- package/dist/browser/{browser/storage.js → storage.js} +20 -33
- package/dist/browser/storage.js.map +1 -0
- package/dist/browser/{browser/wasm.js → wasm.js} +1 -1
- package/dist/node/client.d.ts +107 -0
- package/dist/node/client.d.ts.map +1 -0
- package/dist/node/{node/client.js → client.js} +138 -112
- package/dist/node/client.js.map +1 -0
- package/dist/node/formats/coco.d.ts +62 -0
- package/dist/node/formats/coco.d.ts.map +1 -0
- package/dist/node/{node/formats → formats}/coco.js +94 -8
- package/dist/node/formats/coco.js.map +1 -0
- package/dist/node/{node/formats → formats}/coco_panoptic.d.ts +1 -1
- package/dist/node/{node/formats → formats}/coco_panoptic.d.ts.map +1 -1
- package/dist/node/{node/formats → formats}/coco_panoptic.js +1 -0
- package/dist/node/formats/coco_panoptic.js.map +1 -0
- package/dist/node/formats/lerobot.d.ts +84 -0
- package/dist/node/formats/lerobot.d.ts.map +1 -0
- package/dist/node/{node/formats → formats}/lerobot.js +61 -1
- package/dist/node/formats/lerobot.js.map +1 -0
- package/dist/node/{node/index.d.ts → index.d.ts} +12 -12
- package/dist/node/{node/index.js → index.js} +8 -8
- package/dist/node/normalizers/binary.d.ts +3 -0
- package/dist/node/{node/normalizers → normalizers}/binary.js +1 -1
- package/dist/node/{node/normalizers → normalizers}/binary.js.map +1 -1
- package/dist/node/{node/normalizers → normalizers}/dict.d.ts +1 -1
- package/dist/node/{node/normalizers → normalizers}/files.d.ts +1 -1
- package/dist/node/{node/normalizers → normalizers}/files.js +1 -1
- package/dist/node/normalizers/huggingface.d.ts +23 -0
- package/dist/node/normalizers/huggingface.d.ts.map +1 -0
- package/dist/node/{node/normalizers → normalizers}/huggingface.js +29 -0
- package/dist/node/normalizers/huggingface.js.map +1 -0
- package/dist/node/normalizers/image.d.ts +3 -0
- package/dist/node/{node/normalizers → normalizers}/image.js +1 -1
- package/dist/node/{node/normalizers → normalizers}/image.js.map +1 -1
- package/dist/node/normalizers/pdf.d.ts +3 -0
- package/dist/node/{node/normalizers → normalizers}/pdf.js +2 -2
- package/dist/node/{node/normalizers → normalizers}/pdf.js.map +1 -1
- package/dist/node/{node/normalizers → normalizers}/text.d.ts +1 -1
- package/dist/node/{node/normalizers → normalizers}/text.js +1 -1
- package/dist/node/{node/normalizers → normalizers}/text.js.map +1 -1
- package/dist/node/{node/normalizers → normalizers}/video.d.ts +1 -1
- package/dist/node/{node/normalizers → normalizers}/video.js +2 -2
- package/dist/node/{node/normalizers → normalizers}/video.js.map +1 -1
- package/dist/node/shared/credentials.d.ts.map +1 -1
- package/dist/node/shared/credentials.js +4 -6
- package/dist/node/shared/credentials.js.map +1 -1
- package/dist/node/shared/database.d.ts +3 -1
- package/dist/node/shared/database.d.ts.map +1 -1
- package/dist/node/shared/dlref.d.ts +3 -0
- package/dist/node/shared/dlref.d.ts.map +1 -1
- package/dist/node/shared/dlref.js +3 -0
- package/dist/node/shared/dlref.js.map +1 -1
- package/dist/node/shared/schema.d.ts.map +1 -1
- package/dist/node/shared/schema.js +31 -20
- package/dist/node/shared/schema.js.map +1 -1
- package/dist/node/shared/token.d.ts.map +1 -1
- package/dist/node/shared/token.js +1 -17
- package/dist/node/shared/token.js.map +1 -1
- package/dist/node/{node/storage.d.ts → storage.d.ts} +5 -0
- package/dist/node/storage.d.ts.map +1 -0
- package/dist/node/{node/storage.js → storage.js} +137 -16
- package/dist/node/storage.js.map +1 -0
- package/dist/node/{node/wasm.js → wasm.js} +1 -1
- package/package.json +17 -13
- package/wasm/browser/deeplake_browser.js +1 -1
- package/wasm/browser/deeplake_browser.wasm +0 -0
- package/wasm/node/deeplake_node.js +1 -1
- package/wasm/node/deeplake_node.wasm +0 -0
- package/dist/browser/browser/storage.js.map +0 -1
- package/dist/node/node/client.d.ts +0 -49
- package/dist/node/node/client.d.ts.map +0 -1
- package/dist/node/node/client.js.map +0 -1
- package/dist/node/node/formats/coco.d.ts +0 -24
- package/dist/node/node/formats/coco.d.ts.map +0 -1
- package/dist/node/node/formats/coco.js.map +0 -1
- package/dist/node/node/formats/coco_panoptic.js.map +0 -1
- package/dist/node/node/formats/lerobot.d.ts +0 -42
- package/dist/node/node/formats/lerobot.d.ts.map +0 -1
- package/dist/node/node/formats/lerobot.js.map +0 -1
- package/dist/node/node/normalizers/binary.d.ts +0 -3
- package/dist/node/node/normalizers/huggingface.d.ts +0 -3
- package/dist/node/node/normalizers/huggingface.d.ts.map +0 -1
- package/dist/node/node/normalizers/huggingface.js.map +0 -1
- package/dist/node/node/normalizers/image.d.ts +0 -3
- package/dist/node/node/normalizers/pdf.d.ts +0 -3
- package/dist/node/node/storage.d.ts.map +0 -1
- package/dist/node/node/storage.js.map +0 -1
- package/dist/wasm/node/deeplake_node.js +0 -3
- package/dist/wasm/node/deeplake_node.wasm +0 -0
- package/dist/wasm/node/deeplake_node.worker.js +0 -5
- package/wasm/browser/deeplake_browser.worker.js +0 -5
- package/wasm/node/deeplake_node.worker.js +0 -5
- /package/dist/browser/{browser/client.d.ts.map → client.d.ts.map} +0 -0
- /package/dist/browser/{browser/client.js.map → client.js.map} +0 -0
- /package/dist/browser/{browser/index.d.ts.map → index.d.ts.map} +0 -0
- /package/dist/browser/{browser/index.js.map → index.js.map} +0 -0
- /package/dist/browser/{browser/storage.d.ts → storage.d.ts} +0 -0
- /package/dist/browser/{browser/wasm.d.ts → wasm.d.ts} +0 -0
- /package/dist/browser/{browser/wasm.d.ts.map → wasm.d.ts.map} +0 -0
- /package/dist/browser/{browser/wasm.js.map → wasm.js.map} +0 -0
- /package/dist/node/{node/formats → formats}/index.d.ts +0 -0
- /package/dist/node/{node/formats → formats}/index.d.ts.map +0 -0
- /package/dist/node/{node/formats → formats}/index.js +0 -0
- /package/dist/node/{node/formats → formats}/index.js.map +0 -0
- /package/dist/node/{node/index.d.ts.map → index.d.ts.map} +0 -0
- /package/dist/node/{node/index.js.map → index.js.map} +0 -0
- /package/dist/node/{node/mime.d.ts → mime.d.ts} +0 -0
- /package/dist/node/{node/mime.d.ts.map → mime.d.ts.map} +0 -0
- /package/dist/node/{node/mime.js → mime.js} +0 -0
- /package/dist/node/{node/mime.js.map → mime.js.map} +0 -0
- /package/dist/node/{node/normalizers → normalizers}/binary.d.ts.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/dict.d.ts.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/dict.js +0 -0
- /package/dist/node/{node/normalizers → normalizers}/dict.js.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/files.d.ts.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/files.js.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/image.d.ts.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/index.d.ts +0 -0
- /package/dist/node/{node/normalizers → normalizers}/index.d.ts.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/index.js +0 -0
- /package/dist/node/{node/normalizers → normalizers}/index.js.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/pdf.d.ts.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/text.d.ts.map +0 -0
- /package/dist/node/{node/normalizers → normalizers}/video.d.ts.map +0 -0
- /package/dist/node/{node/wasm.d.ts → wasm.d.ts} +0 -0
- /package/dist/node/{node/wasm.d.ts.map → wasm.d.ts.map} +0 -0
- /package/dist/node/{node/wasm.js.map → wasm.js.map} +0 -0
|
@@ -4,9 +4,9 @@ exports.normalizeVideo = exports.normalizeImage = exports.normalizeBinary = expo
|
|
|
4
4
|
exports.LeRobotEpisodes = exports.LeRobotFrames = exports.LeRobotTasks = exports.CocoPanoptic = exports.Coco = exports.normalizeHuggingface = exports.normalizeFiles = exports.normalizePdf = void 0;
|
|
5
5
|
var client_1 = require("./client");
|
|
6
6
|
Object.defineProperty(exports, "ManagedClient", { enumerable: true, get: function () { return client_1.ManagedClient; } });
|
|
7
|
-
var database_1 = require("
|
|
7
|
+
var database_1 = require("./shared/database");
|
|
8
8
|
Object.defineProperty(exports, "ManagedQueryBuilder", { enumerable: true, get: function () { return database_1.ManagedQueryBuilder; } });
|
|
9
|
-
var errors_1 = require("
|
|
9
|
+
var errors_1 = require("./shared/errors");
|
|
10
10
|
Object.defineProperty(exports, "ManagedServiceError", { enumerable: true, get: function () { return errors_1.ManagedServiceError; } });
|
|
11
11
|
Object.defineProperty(exports, "AuthError", { enumerable: true, get: function () { return errors_1.AuthError; } });
|
|
12
12
|
Object.defineProperty(exports, "TokenError", { enumerable: true, get: function () { return errors_1.TokenError; } });
|
|
@@ -14,22 +14,22 @@ Object.defineProperty(exports, "CredentialsError", { enumerable: true, get: func
|
|
|
14
14
|
Object.defineProperty(exports, "IngestError", { enumerable: true, get: function () { return errors_1.IngestError; } });
|
|
15
15
|
Object.defineProperty(exports, "TableError", { enumerable: true, get: function () { return errors_1.TableError; } });
|
|
16
16
|
Object.defineProperty(exports, "WorkspaceError", { enumerable: true, get: function () { return errors_1.WorkspaceError; } });
|
|
17
|
-
var token_1 = require("
|
|
17
|
+
var token_1 = require("./shared/token");
|
|
18
18
|
Object.defineProperty(exports, "extractOrgId", { enumerable: true, get: function () { return token_1.extractOrgId; } });
|
|
19
19
|
Object.defineProperty(exports, "tryExtractOrgId", { enumerable: true, get: function () { return token_1.tryExtractOrgId; } });
|
|
20
20
|
Object.defineProperty(exports, "fetchOrgIdFromApi", { enumerable: true, get: function () { return token_1.fetchOrgIdFromApi; } });
|
|
21
|
-
var api_1 = require("
|
|
21
|
+
var api_1 = require("./shared/api");
|
|
22
22
|
Object.defineProperty(exports, "apiRequest", { enumerable: true, get: function () { return api_1.apiRequest; } });
|
|
23
|
-
var credentials_1 = require("
|
|
23
|
+
var credentials_1 = require("./shared/credentials");
|
|
24
24
|
Object.defineProperty(exports, "getStorageCreds", { enumerable: true, get: function () { return credentials_1.getStorageCreds; } });
|
|
25
25
|
Object.defineProperty(exports, "applyStorageCreds", { enumerable: true, get: function () { return credentials_1.applyStorageCreds; } });
|
|
26
|
-
var schema_1 = require("
|
|
26
|
+
var schema_1 = require("./shared/schema");
|
|
27
27
|
Object.defineProperty(exports, "SCHEMA_TO_PG", { enumerable: true, get: function () { return schema_1.SCHEMA_TO_PG; } });
|
|
28
28
|
Object.defineProperty(exports, "SCHEMA_TO_DEEPLAKE", { enumerable: true, get: function () { return schema_1.SCHEMA_TO_DEEPLAKE; } });
|
|
29
29
|
Object.defineProperty(exports, "inferType", { enumerable: true, get: function () { return schema_1.inferType; } });
|
|
30
30
|
Object.defineProperty(exports, "buildPgSchema", { enumerable: true, get: function () { return schema_1.buildPgSchema; } });
|
|
31
31
|
Object.defineProperty(exports, "ensureColumns", { enumerable: true, get: function () { return schema_1.ensureColumns; } });
|
|
32
|
-
var dlref_1 = require("
|
|
32
|
+
var dlref_1 = require("./shared/dlref");
|
|
33
33
|
Object.defineProperty(exports, "isDataRef", { enumerable: true, get: function () { return dlref_1.isDataRef; } });
|
|
34
34
|
Object.defineProperty(exports, "parseDataRef", { enumerable: true, get: function () { return dlref_1.parseDataRef; } });
|
|
35
35
|
var mime_1 = require("./mime");
|
|
@@ -38,7 +38,7 @@ var wasm_1 = require("./wasm");
|
|
|
38
38
|
Object.defineProperty(exports, "initializeWasm", { enumerable: true, get: function () { return wasm_1.initializeWasm; } });
|
|
39
39
|
Object.defineProperty(exports, "deinitializeWasm", { enumerable: true, get: function () { return wasm_1.deinitializeWasm; } });
|
|
40
40
|
Object.defineProperty(exports, "BUNDLED_WASM_PATH", { enumerable: true, get: function () { return wasm_1.BUNDLED_WASM_PATH; } });
|
|
41
|
-
var wasm_common_1 = require("
|
|
41
|
+
var wasm_common_1 = require("./shared/wasm-common");
|
|
42
42
|
Object.defineProperty(exports, "deeplakeOpen", { enumerable: true, get: function () { return wasm_common_1.deeplakeOpen; } });
|
|
43
43
|
Object.defineProperty(exports, "deeplakeCreate", { enumerable: true, get: function () { return wasm_common_1.deeplakeCreate; } });
|
|
44
44
|
Object.defineProperty(exports, "deeplakeExists", { enumerable: true, get: function () { return wasm_common_1.deeplakeExists; } });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"binary.js","sourceRoot":"","sources":["../../../../src/node/normalizers/binary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,0CAWC;AAhBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,eAAe,CAAC,QAAgB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,
|
|
1
|
+
{"version":3,"file":"binary.js","sourceRoot":"","sources":["../../../../src/node/normalizers/binary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,0CAWC;AAhBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,eAAe,CAAC,QAAgB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACzB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,IAAI,EAAE,CAAC,IAAI,CAAC;QACZ,QAAQ,EAAE,CAAC,QAAQ,CAAC;KACrB,CAAC;AACJ,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type { Batch } from '
|
|
1
|
+
import type { Batch } from '.././shared/types';
|
|
2
2
|
export declare function normalizeFiles(files: string[], chunkSize: number, chunkOverlap: number, concurrency?: number): AsyncGenerator<Batch>;
|
|
3
3
|
//# sourceMappingURL=files.d.ts.map
|
|
@@ -36,7 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.normalizeFiles = normalizeFiles;
|
|
37
37
|
const path = __importStar(require("path"));
|
|
38
38
|
const fs = __importStar(require("fs"));
|
|
39
|
-
const errors_1 = require("
|
|
39
|
+
const errors_1 = require(".././shared/errors");
|
|
40
40
|
const image_1 = require("./image");
|
|
41
41
|
const text_1 = require("./text");
|
|
42
42
|
const binary_1 = require("./binary");
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HuggingFace dataset normalizer for Node.js.
|
|
3
|
+
*
|
|
4
|
+
* Streams a HuggingFace dataset via the `@huggingface/hub` or `datasets`
|
|
5
|
+
* package and yields column-oriented batches.
|
|
6
|
+
*
|
|
7
|
+
* This is a dynamic-import normalizer — it will throw a helpful error
|
|
8
|
+
* if no HuggingFace library is available.
|
|
9
|
+
*/
|
|
10
|
+
import type { Batch } from '.././shared/types';
|
|
11
|
+
/**
|
|
12
|
+
* Normalize a HuggingFace dataset into column-oriented batches.
|
|
13
|
+
*
|
|
14
|
+
* Attempts to load the dataset using the `datasets` npm package first,
|
|
15
|
+
* falling back to `@huggingface/hub`. If neither is available, throws
|
|
16
|
+
* with installation instructions.
|
|
17
|
+
*
|
|
18
|
+
* @param datasetName - HuggingFace dataset identifier (e.g. "cifar10").
|
|
19
|
+
* @param batchSize - Rows per batch (default 100).
|
|
20
|
+
* @param split - Dataset split (default "train").
|
|
21
|
+
*/
|
|
22
|
+
export declare function normalizeHuggingface(datasetName: string, batchSize?: number, split?: string): AsyncGenerator<Batch>;
|
|
23
|
+
//# sourceMappingURL=huggingface.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"huggingface.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/huggingface.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,wBAAuB,oBAAoB,CACzC,WAAW,EAAE,MAAM,EACnB,SAAS,GAAE,MAAY,EACvB,KAAK,GAAE,MAAgB,GACtB,cAAc,CAAC,KAAK,CAAC,CAgDvB"}
|
|
@@ -1,4 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* HuggingFace dataset normalizer for Node.js.
|
|
4
|
+
*
|
|
5
|
+
* Streams a HuggingFace dataset via the `@huggingface/hub` or `datasets`
|
|
6
|
+
* package and yields column-oriented batches.
|
|
7
|
+
*
|
|
8
|
+
* This is a dynamic-import normalizer — it will throw a helpful error
|
|
9
|
+
* if no HuggingFace library is available.
|
|
10
|
+
*/
|
|
2
11
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
12
|
if (k2 === undefined) k2 = k;
|
|
4
13
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
@@ -34,10 +43,23 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
43
|
})();
|
|
35
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
45
|
exports.normalizeHuggingface = normalizeHuggingface;
|
|
46
|
+
/**
|
|
47
|
+
* Normalize a HuggingFace dataset into column-oriented batches.
|
|
48
|
+
*
|
|
49
|
+
* Attempts to load the dataset using the `datasets` npm package first,
|
|
50
|
+
* falling back to `@huggingface/hub`. If neither is available, throws
|
|
51
|
+
* with installation instructions.
|
|
52
|
+
*
|
|
53
|
+
* @param datasetName - HuggingFace dataset identifier (e.g. "cifar10").
|
|
54
|
+
* @param batchSize - Rows per batch (default 100).
|
|
55
|
+
* @param split - Dataset split (default "train").
|
|
56
|
+
*/
|
|
37
57
|
async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'train') {
|
|
58
|
+
// Try loading via @huggingface/hub (most common in Node.js)
|
|
38
59
|
let rows;
|
|
39
60
|
try {
|
|
40
61
|
const hub = require('@huggingface/hub');
|
|
62
|
+
// @huggingface/hub provides a parquet-based streaming API
|
|
41
63
|
if (typeof hub.listDatasetFiles === 'function') {
|
|
42
64
|
rows = streamViaHub(hub, datasetName, split);
|
|
43
65
|
}
|
|
@@ -46,6 +68,7 @@ async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'trai
|
|
|
46
68
|
}
|
|
47
69
|
}
|
|
48
70
|
catch {
|
|
71
|
+
// Fallback: try to use a Python subprocess
|
|
49
72
|
try {
|
|
50
73
|
rows = streamViaPython(datasetName, split);
|
|
51
74
|
}
|
|
@@ -59,6 +82,7 @@ async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'trai
|
|
|
59
82
|
for await (const item of rows) {
|
|
60
83
|
const row = {};
|
|
61
84
|
for (const [col, val] of Object.entries(item)) {
|
|
85
|
+
// Convert Buffer/Uint8Array values to Buffer for consistency
|
|
62
86
|
if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
|
|
63
87
|
row[col] = Buffer.from(val);
|
|
64
88
|
}
|
|
@@ -77,6 +101,9 @@ async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'trai
|
|
|
77
101
|
}
|
|
78
102
|
}
|
|
79
103
|
async function* streamViaHub(hub, datasetName, split) {
|
|
104
|
+
// Use the hub's dataset viewer API for streaming
|
|
105
|
+
const { createRepo, downloadFile } = hub;
|
|
106
|
+
// Try the datasets server API (works for most public datasets)
|
|
80
107
|
const baseUrl = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(datasetName)}&config=default&split=${encodeURIComponent(split)}`;
|
|
81
108
|
let offset = 0;
|
|
82
109
|
const length = 100;
|
|
@@ -125,6 +152,7 @@ for item in ds:
|
|
|
125
152
|
if (!trimmed)
|
|
126
153
|
continue;
|
|
127
154
|
const row = JSON.parse(trimmed);
|
|
155
|
+
// Decode base64-encoded bytes back to Buffer
|
|
128
156
|
for (const [key, val] of Object.entries(row)) {
|
|
129
157
|
if (val &&
|
|
130
158
|
typeof val === 'object' &&
|
|
@@ -134,6 +162,7 @@ for item in ds:
|
|
|
134
162
|
}
|
|
135
163
|
yield row;
|
|
136
164
|
}
|
|
165
|
+
// Wait for process to exit
|
|
137
166
|
await new Promise((resolve, reject) => {
|
|
138
167
|
proc.on('close', (code) => {
|
|
139
168
|
if (code !== 0) {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"huggingface.js","sourceRoot":"","sources":["../../../../src/node/normalizers/huggingface.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAeH,oDAoDC;AA/DD;;;;;;;;;;GAUG;AACI,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,WAAmB,EACnB,YAAoB,GAAG,EACvB,QAAgB,OAAO;IAEvB,4DAA4D;IAC5D,IAAI,IAA4C,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,CAAC,kBAAkB,CAAQ,CAAC;QAC/C,0DAA0D;QAC1D,IAAI,OAAO,GAAG,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;YAC/C,IAAI,GAAG,YAAY,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2CAA2C;QAC3C,IAAI,CAAC;YACH,IAAI,GAAG,eAAe,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,6DAA6D;gBAC7D,kCAAkC;gBAClC,0DAA0D,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAA8B,EAAE,CAAC;IAE5C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,6DAA6D;YAC7D,IAAI,GAAG,YAAY,UAAU,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YACjB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;YACzB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,YAAY,CAC1B,GAAQ,EACR,WAAmB,EACnB,KAAa;IAEb,iDAAiD;IACjD,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC;IAEzC,+DAA+D;IAC/D,MAAM,OAAO,GAAG,uDAAuD,kBAAkB,CAAC,WAAW,CAAC,yBAAyB,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IAE3J,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,GAAG,CAAC;IAEnB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,GAAG,OAAO,WAAW,MAAM,WAAW,MAAM,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM;QAEpB,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAQ,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAa,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAEtC,KAAK,MAAM,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9B,MAAM,UAAU,CAAC,GAA8B,CAAC;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM;YAAE,MAAM;QAChC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAC7B,WAAmB,EACnB,KAAa;IAEb,MAAM,EAAE,KAAK,EAAE,GAAG,wDAAa,eAAe,GAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,wDAAa,UAAU,GAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG;;;qBAGE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,aAAa,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;;;;;;;;;;;;CAY3F,CAAC;IAEA,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE;QAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;KAChC,CAAC,CAAC;IAEH,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAO,EAAE,CAAC,CAAC;IAE7D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;QAE3D,6CAA6C;QAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IACE,GAAG;gBACH,OAAO,GAAG,KAAK,QAAQ;gBACvB,WAAW,IAAK,GAA+B,EAC/C,CAAC;gBACD,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CACnB,GAA6B,CAAC,SAAS,EACxC,QAAQ,CACT,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,2BAA2B;IAC3B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,IAA+B;IAClD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../../../src/node/normalizers/image.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,wCAYC;AAjBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,cAAc,CAAC,QAAgB;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,
|
|
1
|
+
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../../../src/node/normalizers/image.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,wCAYC;AAjBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,cAAc,CAAC,QAAgB;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACzB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;KAC7B,CAAC;AACJ,CAAC"}
|
|
@@ -47,7 +47,7 @@ async function* normalizePdf(filePath) {
|
|
|
47
47
|
catch {
|
|
48
48
|
// Fallback: store whole PDF as binary
|
|
49
49
|
yield {
|
|
50
|
-
|
|
50
|
+
chunk_id: [`${fileId}_0`],
|
|
51
51
|
file_id: [fileId],
|
|
52
52
|
page_index: [0],
|
|
53
53
|
data: [fs.readFileSync(filePath)],
|
|
@@ -64,7 +64,7 @@ async function* normalizePdf(filePath) {
|
|
|
64
64
|
.map((item) => item.str)
|
|
65
65
|
.join(' ');
|
|
66
66
|
yield {
|
|
67
|
-
|
|
67
|
+
chunk_id: [`${fileId}_${pageIdx}`],
|
|
68
68
|
file_id: [fileId],
|
|
69
69
|
page_index: [pageIdx],
|
|
70
70
|
text: [text || `PDF page ${pageIdx + 1}`],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../../src/node/normalizers/pdf.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,oCAsCC;AA3CD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAG1B,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,QAAgB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,IAAI,KAAU,CAAC;IACf,IAAI,CAAC;QACH,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;QACtC,MAAM;YACJ,
|
|
1
|
+
{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../../src/node/normalizers/pdf.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,oCAsCC;AA3CD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAG1B,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,QAAgB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,IAAI,KAAU,CAAC;IACf,IAAI,CAAC;QACH,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;QACtC,MAAM;YACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACzB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,CAAC,CAAC;YACf,IAAI,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,EAAE,CAAC,QAAQ,QAAQ,EAAE,CAAC;SAC3B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC;IAEtD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK;aAC3B,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;aAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM;YACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,OAAO,EAAE,CAAC;YAClC,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,OAAO,CAAC;YACrB,IAAI,EAAE,CAAC,IAAI,IAAI,YAAY,OAAO,GAAG,CAAC,EAAE,CAAC;SAC1C,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,OAAO,EAAE,CAAC;AAChB,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Batch } from '
|
|
1
|
+
import type { Batch } from '.././shared/types';
|
|
2
2
|
export declare function chunkText(text: string, chunkSize: number, overlap: number): string[];
|
|
3
3
|
export declare function normalizeText(filePath: string, chunkSize: number, chunkOverlap: number): Batch;
|
|
4
4
|
//# sourceMappingURL=text.d.ts.map
|
|
@@ -74,7 +74,7 @@ function normalizeText(filePath, chunkSize, chunkOverlap) {
|
|
|
74
74
|
const chunks = chunkText(content, chunkSize, chunkOverlap);
|
|
75
75
|
const name = filePath.split('/').pop() ?? filePath;
|
|
76
76
|
return {
|
|
77
|
-
|
|
77
|
+
chunk_id: chunks.map((_, i) => `${fileId}_${i}`),
|
|
78
78
|
file_id: Array(chunks.length).fill(fileId),
|
|
79
79
|
chunk_index: chunks.map((_, i) => i),
|
|
80
80
|
text: chunks.length > 0 ? chunks : [`Text: ${name}`],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/node/normalizers/text.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,8BAwCC;AAED,sCAgBC;AA9DD,uCAAyB;AACzB,+CAAiC;AAGjC,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,OAAe;IAEf,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,KAAK,GAAG,SAAS,CAAC;QAE5B,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM;QACR,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7D,IAAI,SAAS,GAAG,GAAG,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;YAC5C,IAAI,GAAG,IAAI,WAAW,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;gBACtC,SAAS,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;QACrC,KAAK,GAAG,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,QAAgB,EAChB,SAAiB,EACjB,YAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IAEnD,OAAO;QACL,
|
|
1
|
+
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/node/normalizers/text.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,8BAwCC;AAED,sCAgBC;AA9DD,uCAAyB;AACzB,+CAAiC;AAGjC,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,OAAe;IAEf,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,KAAK,GAAG,SAAS,CAAC;QAE5B,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM;QACR,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7D,IAAI,SAAS,GAAG,GAAG,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;YAC5C,IAAI,GAAG,IAAI,WAAW,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;gBACtC,SAAS,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;QACrC,KAAK,GAAG,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,QAAgB,EAChB,SAAiB,EACjB,YAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IAEnD,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC;QAChD,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}
|
|
@@ -61,7 +61,7 @@ async function* normalizeVideo(filePath, chunkInterval = 10) {
|
|
|
61
61
|
const duration = await probeDuration(filePath);
|
|
62
62
|
if (duration <= 0) {
|
|
63
63
|
yield {
|
|
64
|
-
|
|
64
|
+
chunk_id: [`${fileId}_0`],
|
|
65
65
|
file_id: [fileId],
|
|
66
66
|
chunk_index: [0],
|
|
67
67
|
start_time: [0.0],
|
|
@@ -101,7 +101,7 @@ async function* normalizeVideo(filePath, chunkInterval = 10) {
|
|
|
101
101
|
}
|
|
102
102
|
const videoData = fs.existsSync(segPath) ? fs.readFileSync(segPath) : null;
|
|
103
103
|
yield {
|
|
104
|
-
|
|
104
|
+
chunk_id: [`${fileId}_${idx}`],
|
|
105
105
|
file_id: [fileId],
|
|
106
106
|
chunk_index: [idx],
|
|
107
107
|
start_time: [Math.round(start * 1000) / 1000],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"video.js","sourceRoot":"","sources":["../../../../src/node/normalizers/video.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,wCA2EC;AAnGD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AACjC,iDAAyC;AACzC,+BAAiC;AAGjC,MAAM,aAAa,GAAG,IAAA,gBAAS,EAAC,wBAAQ,CAAC,CAAC;AAE1C,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE;YAChD,IAAI,EAAE,OAAO;YACb,eAAe,EAAE,MAAM;YACvB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,IAAI,GAAG,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,QAAgB,EAChB,gBAAwB,EAAE;IAE1B,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;IAE/C,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM;YACJ,
|
|
1
|
+
{"version":3,"file":"video.js","sourceRoot":"","sources":["../../../../src/node/normalizers/video.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,wCA2EC;AAnGD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AACjC,iDAAyC;AACzC,+BAAiC;AAGjC,MAAM,aAAa,GAAG,IAAA,gBAAS,EAAC,wBAAQ,CAAC,CAAC;AAE1C,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE;YAChD,IAAI,EAAE,OAAO;YACb,eAAe,EAAE,MAAM;YACvB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,IAAI,GAAG,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,QAAgB,EAChB,gBAAwB,EAAE;IAE1B,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;IAE/C,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM;YACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACzB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,WAAW,EAAE,CAAC,CAAC,CAAC;YAChB,UAAU,EAAE,CAAC,GAAG,CAAC;YACjB,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,UAAU,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvC,SAAS,EAAE,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;SAC7B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,wDAAa,IAAI,GAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;IAE5B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,GAAG,GAAG,aAAa,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,KAAK,GAAG,WAAW,CAAC;QAEhC,MAAM,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAC9F,EAAE,OAAO,EAAE,OAAO,EAAE,CACrB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EACrG,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAC;gBACF,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7B,SAAS,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YAED,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAE3E,MAAM;gBACJ,QAAQ,EAAE,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC;gBAC9B,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBAC7C,QAAQ,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBACzC,UAAU,EAAE,CAAC,SAAS,CAAC;gBACvB,SAAS,EAAE,CAAC,SAAS,CAAC;gBACtB,IAAI,EAAE,CAAC,iBAAiB,GAAG,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;aAC1E,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE,MAAoB,GACzB,OAAO,CAAC,YAAY,CAAC,CAYvB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE,MAAoB,GACzB,OAAO,CAAC,YAAY,CAAC,CAYvB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAM3D"}
|
|
@@ -17,12 +17,10 @@ async function getStorageCreds(baseUrl, token, orgId, workspaceId, mode = 'readw
|
|
|
17
17
|
};
|
|
18
18
|
}
|
|
19
19
|
function applyStorageCreds(creds) {
|
|
20
|
-
if (
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
process.env.AWS_SESSION_TOKEN = creds.aws_session_token ?? '';
|
|
25
|
-
}
|
|
20
|
+
if (creds.aws_access_key_id) {
|
|
21
|
+
process.env.AWS_ACCESS_KEY_ID = creds.aws_access_key_id;
|
|
22
|
+
process.env.AWS_SECRET_ACCESS_KEY = creds.aws_secret_access_key;
|
|
23
|
+
process.env.AWS_SESSION_TOKEN = creds.aws_session_token ?? '';
|
|
26
24
|
}
|
|
27
25
|
}
|
|
28
26
|
//# sourceMappingURL=credentials.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":";;AAGA,0CAkBC;AAED,
|
|
1
|
+
{"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../../src/shared/credentials.ts"],"names":[],"mappings":";;AAGA,0CAkBC;AAED,8CAMC;AA7BD,+BAAmC;AAG5B,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,KAAa,EACb,KAAa,EACb,WAAmB,EACnB,OAAe,WAAW;IAE1B,MAAM,IAAI,GAAG,MAAM,IAAA,gBAAU,EAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE;QACnD,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,eAAe,WAAW,uBAAuB;QACvD,IAAI,EAAE,EAAE,IAAI,EAAE;KACf,CAAC,CAAC;IACH,MAAM,KAAK,GAAI,IAAI,CAAC,KAAgC,IAAI,IAAI,CAAC;IAC7D,OAAO;QACL,iBAAiB,EAAE,KAAK,CAAC,iBAA2B;QACpD,qBAAqB,EAAE,KAAK,CAAC,qBAA+B;QAC5D,iBAAiB,EAAE,KAAK,CAAC,iBAAuC;KACjE,CAAC;AACJ,CAAC;AAED,SAAgB,iBAAiB,CAAC,KAAmB;IACnD,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,KAAK,CAAC,qBAAqB,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAChE,CAAC;AACH,CAAC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { QueryRow } from './types';
|
|
2
2
|
export interface QueryExecutor {
|
|
3
|
-
query(sql: string, params?: unknown[]
|
|
3
|
+
query(sql: string, params?: unknown[], options?: {
|
|
4
|
+
timeoutMs?: number;
|
|
5
|
+
}): Promise<QueryRow[]>;
|
|
4
6
|
}
|
|
5
7
|
export declare class ManagedQueryBuilder {
|
|
6
8
|
private _executor;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/shared/database.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/shared/database.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CAC/F;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,WAAW,CAAkD;IACrE,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,UAAU,CAAuB;gBAE7B,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM;IAKlD,MAAM,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI;IAKlC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI;IAKpD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK7B,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKtB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKvB,QAAQ,IAAI;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;KAAE;IA+B/C,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKpC,QAAQ,IAAI,MAAM;CAInB"}
|
|
@@ -11,6 +11,9 @@ export interface DataRef {
|
|
|
11
11
|
columnId: number;
|
|
12
12
|
rowId: bigint;
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Check if a value is a DLREF data reference.
|
|
16
|
+
*/
|
|
14
17
|
export declare function isDataRef(value: unknown): boolean;
|
|
15
18
|
/**
|
|
16
19
|
* Parse a DLREF data reference string into its components.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dlref.d.ts","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAID,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CASjD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,OAAO,CAuBhE"}
|
|
1
|
+
{"version":3,"file":"dlref.d.ts","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAID;;GAEG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CASjD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,OAAO,CAuBhE"}
|
|
@@ -11,6 +11,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
11
11
|
exports.isDataRef = isDataRef;
|
|
12
12
|
exports.parseDataRef = parseDataRef;
|
|
13
13
|
const DLREF_PREFIX = 'DLREF:';
|
|
14
|
+
/**
|
|
15
|
+
* Check if a value is a DLREF data reference.
|
|
16
|
+
*/
|
|
14
17
|
function isDataRef(value) {
|
|
15
18
|
if (typeof value === 'string') {
|
|
16
19
|
return value.startsWith(DLREF_PREFIX);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dlref.js","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;
|
|
1
|
+
{"version":3,"file":"dlref.js","sourceRoot":"","sources":["../../../src/shared/dlref.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AAaH,8BASC;AAQD,oCAuBC;AA7CD,MAAM,YAAY,GAAG,QAAQ,CAAC;AAE9B;;GAEG;AACH,SAAgB,SAAS,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9D,OAAO,MAAM,KAAK,YAAY,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,SAAgB,YAAY,CAAC,KAA0B;IACrD,IAAI,GAAW,CAAC;IAChB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,GAAG,GAAG,KAAK,CAAC;IACd,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,+CAA+C,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACjB,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAChC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAExD,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqB/C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqBrD,CAAC;AAEF,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAExD,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqB/C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqBrD,CAAC;AAEF,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAmBhD;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,KAAK,EACZ,MAAM,CAAC,EAAE,SAAS,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAwCxB;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,GAAG,EACZ,KAAK,EAAE,KAAK,EACZ,MAAM,CAAC,EAAE,SAAS,GACjB,IAAI,CAiDN"}
|
|
@@ -14,17 +14,17 @@ exports.SCHEMA_TO_PG = {
|
|
|
14
14
|
FLOAT64: 'DOUBLE PRECISION',
|
|
15
15
|
BOOL: 'BOOLEAN',
|
|
16
16
|
BINARY: 'BYTEA',
|
|
17
|
-
IMAGE: '
|
|
17
|
+
IMAGE: 'IMAGE',
|
|
18
18
|
VIDEO: 'VIDEO',
|
|
19
19
|
EMBEDDING: 'float4[]',
|
|
20
|
-
SEGMENT_MASK: '
|
|
21
|
-
BINARY_MASK: '
|
|
22
|
-
BOUNDING_BOX: '
|
|
23
|
-
CLASS_LABEL: '
|
|
24
|
-
POLYGON: '
|
|
25
|
-
POINT: '
|
|
26
|
-
MESH: '
|
|
27
|
-
MEDICAL: '
|
|
20
|
+
SEGMENT_MASK: 'SEGMENT_MASK',
|
|
21
|
+
BINARY_MASK: 'BINARY_MASK',
|
|
22
|
+
BOUNDING_BOX: 'BOUNDING_BOX',
|
|
23
|
+
CLASS_LABEL: 'CLASS_LABEL',
|
|
24
|
+
POLYGON: 'DEEPLAKE_POLYGON',
|
|
25
|
+
POINT: 'DEEPLAKE_POINT',
|
|
26
|
+
MESH: 'MESH',
|
|
27
|
+
MEDICAL: 'MEDICAL',
|
|
28
28
|
};
|
|
29
29
|
exports.SCHEMA_TO_DEEPLAKE = {
|
|
30
30
|
TEXT: 'text',
|
|
@@ -36,17 +36,17 @@ exports.SCHEMA_TO_DEEPLAKE = {
|
|
|
36
36
|
FLOAT64: 'float64',
|
|
37
37
|
BOOL: 'bool',
|
|
38
38
|
BINARY: 'bytes',
|
|
39
|
-
IMAGE: '
|
|
39
|
+
IMAGE: 'bytes',
|
|
40
40
|
VIDEO: 'bytes',
|
|
41
41
|
EMBEDDING: 'float32',
|
|
42
|
-
SEGMENT_MASK: '
|
|
43
|
-
BINARY_MASK: '
|
|
44
|
-
BOUNDING_BOX: '
|
|
45
|
-
CLASS_LABEL: '
|
|
46
|
-
POLYGON: '
|
|
47
|
-
POINT: '
|
|
48
|
-
MESH: '
|
|
49
|
-
MEDICAL: '
|
|
42
|
+
SEGMENT_MASK: 'bytes',
|
|
43
|
+
BINARY_MASK: 'bytes',
|
|
44
|
+
BOUNDING_BOX: 'float32',
|
|
45
|
+
CLASS_LABEL: 'int32',
|
|
46
|
+
POLYGON: 'bytes',
|
|
47
|
+
POINT: 'float32',
|
|
48
|
+
MESH: 'bytes',
|
|
49
|
+
MEDICAL: 'bytes',
|
|
50
50
|
};
|
|
51
51
|
function inferType(value) {
|
|
52
52
|
if (typeof value === 'boolean')
|
|
@@ -58,9 +58,8 @@ function inferType(value) {
|
|
|
58
58
|
return 'TEXT';
|
|
59
59
|
if ((typeof Buffer !== 'undefined' && Buffer.isBuffer(value)) ||
|
|
60
60
|
value instanceof Uint8Array ||
|
|
61
|
-
value instanceof ArrayBuffer)
|
|
61
|
+
value instanceof ArrayBuffer)
|
|
62
62
|
return 'BINARY';
|
|
63
|
-
}
|
|
64
63
|
if (Array.isArray(value) &&
|
|
65
64
|
value.length > 0 &&
|
|
66
65
|
value.every((x) => typeof x === 'number')) {
|
|
@@ -91,6 +90,18 @@ function buildPgSchema(batch, schema, format) {
|
|
|
91
90
|
}
|
|
92
91
|
}
|
|
93
92
|
}
|
|
93
|
+
// Add _thumbnail columns for each IMAGE column. Thumbnails are
|
|
94
|
+
// stored inline in the same table (128×128 JPEG) rather than in
|
|
95
|
+
// a separate thumbnails table. Use BYTEA (not IMAGE domain) since
|
|
96
|
+
// thumbnails are derived data, not primary images.
|
|
97
|
+
for (const [colName, pgType] of Object.entries({ ...pgSchema })) {
|
|
98
|
+
if (pgType.toUpperCase() === 'IMAGE') {
|
|
99
|
+
const thumbCol = `${colName}_thumbnail`;
|
|
100
|
+
if (!(thumbCol in pgSchema)) {
|
|
101
|
+
pgSchema[thumbCol] = 'IMAGE';
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
94
105
|
return pgSchema;
|
|
95
106
|
}
|
|
96
107
|
function ensureColumns(dataset, batch, schema) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":";;;AAgDA,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/shared/schema.ts"],"names":[],"mappings":";;;AAgDA,8BAmBC;AAED,sCA4CC;AAED,sCAqDC;AAtKY,QAAA,YAAY,GAA2B;IAClD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,QAAQ;IACb,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,kBAAkB;IACzB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,kBAAkB;IAC3B,IAAI,EAAE,SAAS;IACf,MAAM,EAAE,OAAO;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,cAAc;IAC5B,WAAW,EAAE,aAAa;IAC1B,YAAY,EAAE,cAAc;IAC5B,WAAW,EAAE,aAAa;IAC1B,OAAO,EAAE,kBAAkB;IAC3B,KAAK,EAAE,gBAAgB;IACvB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;CACnB,CAAC;AAEW,QAAA,kBAAkB,GAA2B;IACxD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,OAAO;IACZ,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,SAAS;IAClB,OAAO,EAAE,SAAS;IAClB,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,OAAO;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,SAAS;IACpB,YAAY,EAAE,OAAO;IACrB,WAAW,EAAE,OAAO;IACpB,YAAY,EAAE,SAAS;IACvB,WAAW,EAAE,OAAO;IACpB,OAAO,EAAE,OAAO;IAChB,KAAK,EAAE,SAAS;IAChB,IAAI,EAAE,OAAO;IACb,OAAO,EAAE,OAAO;CACjB,CAAC;AAEF,SAAgB,SAAS,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACvD,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7C,IACE,CAAC,OAAO,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzD,KAAK,YAAY,UAAU;QAC3B,KAAK,YAAY,WAAW;QAC5B,OAAO,QAAQ,CAAC;IAClB,IACE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACpB,KAAK,CAAC,MAAM,GAAG,CAAC;QAChB,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EACzC,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,KAAY,EACZ,MAAkB,EAClB,MAAe;IAEf,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAE5C,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,QAAQ,KAAK,MAAM;gBAAE,SAAS;YAClC,QAAQ,CAAC,OAAO,CAAC,GAAG,oBAAY,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/E,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,QAAQ,CAAC,OAAO,CAAC,GAAG,oBAAY,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;IACvD,CAAC;IAED,8EAA8E;IAC9E,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YACnE,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,QAAQ,CAAC,OAAO,CAAC,GAAG,UAAU,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,gEAAgE;IAChE,mEAAmE;IACnE,mDAAmD;IACnD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAC,GAAG,QAAQ,EAAC,CAAC,EAAE,CAAC;QAC9D,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,GAAG,OAAO,YAAY,CAAC;YACxC,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAgB,aAAa,CAC3B,OAAY,EACZ,KAAY,EACZ,MAAkB;IAElB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,SAAS,GAAQ,IAAI,CAAC;IAC1B,IAAI,CAAC;QACH,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC;YAC/B,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBAC5C,uCAAuC;gBACvC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACxB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACvB,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,UAAU;wBAAE,GAAG,CAAC,MAAM,EAAE,CAAC;gBACrD,CAAC;gBACD,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU;oBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACvD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,KAAK,MAAM,CAAC,IAAI,IAAI;oBAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;YAAS,CAAC;QACT,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxD,IAAI,CAAC;gBAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QAEpC,IAAI,MAAc,CAAC;QACnB,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,QAAQ,KAAK,MAAM;gBAAE,SAAS;YAClC,MAAM,GAAG,0BAAkB,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC/E,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,GAAG,0BAAkB,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC;QAClD,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAAE,MAAM,CAAC,CAAC;QACnE,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":"AAEA,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAyBlD;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM5D;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgBxB"}
|
|
@@ -4,29 +4,13 @@ exports.extractOrgId = extractOrgId;
|
|
|
4
4
|
exports.tryExtractOrgId = tryExtractOrgId;
|
|
5
5
|
exports.fetchOrgIdFromApi = fetchOrgIdFromApi;
|
|
6
6
|
const errors_1 = require("./errors");
|
|
7
|
-
function base64UrlDecode(str) {
|
|
8
|
-
if (typeof Buffer !== 'undefined') {
|
|
9
|
-
return Buffer.from(str, 'base64url').toString('utf-8');
|
|
10
|
-
}
|
|
11
|
-
// Browser fallback using atob + TextDecoder
|
|
12
|
-
let base64 = str.replace(/-/g, '+').replace(/_/g, '/');
|
|
13
|
-
while (base64.length % 4 !== 0) {
|
|
14
|
-
base64 += '=';
|
|
15
|
-
}
|
|
16
|
-
const binary = atob(base64);
|
|
17
|
-
const bytes = new Uint8Array(binary.length);
|
|
18
|
-
for (let i = 0; i < binary.length; i++) {
|
|
19
|
-
bytes[i] = binary.charCodeAt(i);
|
|
20
|
-
}
|
|
21
|
-
return new TextDecoder().decode(bytes);
|
|
22
|
-
}
|
|
23
7
|
function extractOrgId(token) {
|
|
24
8
|
try {
|
|
25
9
|
const parts = token.split('.');
|
|
26
10
|
if (parts.length !== 3) {
|
|
27
11
|
throw new errors_1.TokenError('Invalid token format: expected JWT with 3 parts');
|
|
28
12
|
}
|
|
29
|
-
const decoded =
|
|
13
|
+
const decoded = Buffer.from(parts[1], 'base64url').toString('utf-8');
|
|
30
14
|
const claims = JSON.parse(decoded);
|
|
31
15
|
const orgId = claims.OrgID || claims.org_id;
|
|
32
16
|
if (!orgId) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"token.js","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"token.js","sourceRoot":"","sources":["../../../src/shared/token.ts"],"names":[],"mappings":";;AAEA,oCAyBC;AAED,0CAMC;AAED,8CAmBC;AAxDD,qCAAsC;AAEtC,SAAgB,YAAY,CAAC,KAAa;IACxC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,mBAAU,CAAC,iDAAiD,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC;QAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,mBAAU,CAAC,qCAAqC,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,mBAAU;YAAE,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,YAAY,WAAW,EAAE,CAAC;YAC7B,MAAM,IAAI,mBAAU,CAAC,mCAAmC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,IAAI,mBAAU,CAClB,wCAAwC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAC7E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAgB,eAAe,CAAC,KAAa;IAC3C,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,KAAK,EAAE;YACvC,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,KAAK,EAAE;gBAChC,cAAc,EAAE,kBAAkB;aACnC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA4B,CAAC;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,aAAqC,CAAC;QACxD,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|