@peers-app/peers-sdk 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/data-context.d.ts +4 -4
- package/dist/context/data-context.js +1 -1
- package/dist/context/index.d.ts +3 -3
- package/dist/context/index.js +4 -0
- package/dist/context/user-context-singleton.js +13 -14
- package/dist/context/user-context.d.ts +4 -4
- package/dist/context/user-context.js +48 -31
- package/dist/data/assistants.d.ts +1 -1
- package/dist/data/assistants.js +35 -24
- package/dist/data/change-tracking.d.ts +8 -8
- package/dist/data/change-tracking.js +45 -39
- package/dist/data/channels.js +5 -5
- package/dist/data/data-locks.d.ts +2 -2
- package/dist/data/data-locks.js +21 -23
- package/dist/data/data-locks.test.js +73 -75
- package/dist/data/device-sync-info.d.ts +1 -1
- package/dist/data/device-sync-info.js +4 -4
- package/dist/data/devices.d.ts +1 -1
- package/dist/data/devices.js +9 -12
- package/dist/data/embeddings.js +14 -11
- package/dist/data/files/file-read-stream.d.ts +2 -2
- package/dist/data/files/file-read-stream.js +23 -14
- package/dist/data/files/file-write-stream.d.ts +2 -2
- package/dist/data/files/file-write-stream.js +8 -8
- package/dist/data/files/file.types.d.ts +2 -2
- package/dist/data/files/file.types.js +17 -11
- package/dist/data/files/files.d.ts +6 -6
- package/dist/data/files/files.js +17 -19
- package/dist/data/files/files.test.js +213 -214
- package/dist/data/files/index.d.ts +4 -4
- package/dist/data/files/index.js +4 -4
- package/dist/data/group-member-roles.js +2 -2
- package/dist/data/group-members.d.ts +5 -5
- package/dist/data/group-members.js +27 -18
- package/dist/data/group-members.test.js +73 -73
- package/dist/data/group-permissions.d.ts +3 -3
- package/dist/data/group-permissions.js +13 -11
- package/dist/data/group-share.d.ts +2 -2
- package/dist/data/group-share.js +29 -24
- package/dist/data/groups.d.ts +4 -4
- package/dist/data/groups.js +27 -19
- package/dist/data/groups.test.js +44 -44
- package/dist/data/index.d.ts +6 -6
- package/dist/data/index.js +6 -6
- package/dist/data/knowledge/peer-types.js +9 -9
- package/dist/data/messages.d.ts +5 -5
- package/dist/data/messages.js +43 -30
- package/dist/data/orm/client-proxy.data-source.d.ts +4 -4
- package/dist/data/orm/client-proxy.data-source.js +10 -12
- package/dist/data/orm/cursor.d.ts +1 -1
- package/dist/data/orm/cursor.js +2 -2
- package/dist/data/orm/cursor.test.js +92 -93
- package/dist/data/orm/data-query.d.ts +3 -3
- package/dist/data/orm/data-query.js +24 -18
- package/dist/data/orm/data-query.mongo.d.ts +1 -1
- package/dist/data/orm/data-query.mongo.js +49 -51
- package/dist/data/orm/data-query.mongo.test.js +173 -204
- package/dist/data/orm/data-query.sqlite.d.ts +1 -1
- package/dist/data/orm/data-query.sqlite.js +84 -73
- package/dist/data/orm/data-query.sqlite.test.js +164 -176
- package/dist/data/orm/data-query.test.js +216 -224
- package/dist/data/orm/decorators.js +3 -3
- package/dist/data/orm/dependency-injection.test.js +53 -56
- package/dist/data/orm/doc.d.ts +4 -4
- package/dist/data/orm/doc.js +17 -21
- package/dist/data/orm/event-registry.d.ts +1 -1
- package/dist/data/orm/event-registry.test.js +16 -16
- package/dist/data/orm/factory.d.ts +2 -2
- package/dist/data/orm/factory.js +33 -33
- package/dist/data/orm/index.d.ts +10 -10
- package/dist/data/orm/index.js +10 -10
- package/dist/data/orm/multi-cursors.d.ts +1 -1
- package/dist/data/orm/multi-cursors.js +6 -6
- package/dist/data/orm/multi-cursors.test.js +152 -144
- package/dist/data/orm/sql.data-source.d.ts +7 -7
- package/dist/data/orm/sql.data-source.js +88 -93
- package/dist/data/orm/sql.data-source.test.js +109 -101
- package/dist/data/orm/subscribable.data-source.d.ts +4 -4
- package/dist/data/orm/subscribable.data-source.js +5 -5
- package/dist/data/orm/table-container-events.test.js +34 -26
- package/dist/data/orm/table-container.d.ts +6 -6
- package/dist/data/orm/table-container.js +33 -21
- package/dist/data/orm/table-container.test.js +64 -53
- package/dist/data/orm/table-definitions.system.d.ts +3 -3
- package/dist/data/orm/table-definitions.system.js +3 -3
- package/dist/data/orm/table-definitions.type.d.ts +5 -5
- package/dist/data/orm/table-dependencies.d.ts +2 -2
- package/dist/data/orm/table.d.ts +5 -5
- package/dist/data/orm/table.event-source.test.js +105 -115
- package/dist/data/orm/table.js +35 -34
- package/dist/data/orm/types.d.ts +3 -3
- package/dist/data/orm/types.js +26 -25
- package/dist/data/orm/types.test.js +166 -92
- package/dist/data/package-permissions.d.ts +1 -1
- package/dist/data/package-permissions.js +2 -2
- package/dist/data/package-version-permissions.d.ts +1 -1
- package/dist/data/package-version-permissions.js +2 -2
- package/dist/data/package-versions.d.ts +9 -9
- package/dist/data/package-versions.js +47 -33
- package/dist/data/packages.d.ts +2 -2
- package/dist/data/packages.js +36 -18
- package/dist/data/packages.utils.d.ts +2 -2
- package/dist/data/packages.utils.js +4 -4
- package/dist/data/persistent-vars.d.ts +15 -15
- package/dist/data/persistent-vars.js +165 -154
- package/dist/data/table-definitions-table.d.ts +5 -5
- package/dist/data/table-definitions-table.js +13 -12
- package/dist/data/tool-tests.js +6 -6
- package/dist/data/tools.js +29 -19
- package/dist/data/user-permissions.d.ts +1 -1
- package/dist/data/user-permissions.js +5 -5
- package/dist/data/user-permissions.test.js +90 -88
- package/dist/data/user-trust-levels.js +10 -10
- package/dist/data/users.d.ts +4 -4
- package/dist/data/users.js +16 -15
- package/dist/data/voice-messages.d.ts +2 -2
- package/dist/data/voice-messages.js +13 -13
- package/dist/data/welcome-modal.pvar.js +3 -1
- package/dist/data/workflow-logs.js +26 -18
- package/dist/data/workflow-runs.d.ts +6 -6
- package/dist/data/workflow-runs.js +70 -44
- package/dist/data/workflows.d.ts +2 -2
- package/dist/data/workflows.js +7 -9
- package/dist/device/binary-peer-connection-v2.d.ts +7 -7
- package/dist/device/binary-peer-connection-v2.js +32 -28
- package/dist/device/binary-peer-connection-v2.test.js +80 -67
- package/dist/device/binary-peer-connection.d.ts +7 -7
- package/dist/device/binary-peer-connection.js +29 -28
- package/dist/device/binary-peer-connection.test.js +35 -31
- package/dist/device/connection.d.ts +5 -5
- package/dist/device/connection.js +59 -48
- package/dist/device/connection.test.js +74 -68
- package/dist/device/device-election.d.ts +2 -2
- package/dist/device/device-election.js +25 -20
- package/dist/device/device-election.test.js +35 -36
- package/dist/device/device.d.ts +2 -2
- package/dist/device/device.js +10 -4
- package/dist/device/device.test.js +16 -17
- package/dist/device/get-trust-level-fn.d.ts +2 -2
- package/dist/device/get-trust-level-fn.js +22 -11
- package/dist/device/get-trust-level-fn.test.js +58 -58
- package/dist/device/socket-io-binary-peer.d.ts +1 -1
- package/dist/device/socket-io-binary-peer.js +16 -13
- package/dist/device/socket.type.d.ts +2 -2
- package/dist/device/streamed-socket.d.ts +2 -2
- package/dist/device/streamed-socket.js +8 -8
- package/dist/device/streamed-socket.test.js +40 -40
- package/dist/device/tx-encoding.test.js +77 -77
- package/dist/events.d.ts +1 -1
- package/dist/events.js +5 -2
- package/dist/group-invite/group-invite.js +110 -19
- package/dist/group-invite/group-invite.pvars.d.ts +2 -2
- package/dist/group-invite/group-invite.pvars.js +21 -13
- package/dist/group-invite/group-invite.types.d.ts +1 -1
- package/dist/group-invite/index.d.ts +3 -3
- package/dist/group-invite/index.js +1 -1
- package/dist/index.d.ts +25 -24
- package/dist/index.js +30 -25
- package/dist/keys.d.ts +3 -3
- package/dist/keys.js +31 -30
- package/dist/keys.test.js +69 -61
- package/dist/logging/console-logger.d.ts +1 -1
- package/dist/logging/console-logger.js +35 -40
- package/dist/logging/console-logger.test.js +115 -115
- package/dist/logging/console-logs.table.d.ts +3 -3
- package/dist/logging/console-logs.table.js +28 -23
- package/dist/mentions.js +16 -12
- package/dist/observable.d.ts +2 -2
- package/dist/observable.js +15 -9
- package/dist/observable.test.js +47 -47
- package/dist/package-loader/get-require.js +3 -4
- package/dist/package-loader/package-loader.d.ts +2 -2
- package/dist/package-loader/package-loader.js +52 -34
- package/dist/peers-ui/peers-ui.d.ts +2 -2
- package/dist/peers-ui/peers-ui.js +2 -4
- package/dist/peers-ui/peers-ui.types.d.ts +3 -3
- package/dist/peers-ui/peers-ui.types.js +0 -1
- package/dist/rpc-types.d.ts +61 -59
- package/dist/rpc-types.js +61 -55
- package/dist/serial-json.d.ts +1 -1
- package/dist/serial-json.js +50 -43
- package/dist/serial-json.test.js +22 -22
- package/dist/system-ids.js +8 -8
- package/dist/tools/index.d.ts +1 -1
- package/dist/tools/tools-factory.d.ts +1 -1
- package/dist/tools/tools-factory.js +2 -2
- package/dist/types/assistant-runner-args.d.ts +3 -3
- package/dist/types/peer-device.d.ts +1 -1
- package/dist/types/peers-package.d.ts +3 -3
- package/dist/types/workflow-logger.d.ts +1 -1
- package/dist/types/workflow-run-context.d.ts +4 -4
- package/dist/types/workflow.d.ts +4 -4
- package/dist/types/workflow.js +27 -14
- package/dist/types/zod-types.d.ts +2 -1
- package/dist/types/zod-types.js +9 -3
- package/dist/user-connect/connection-code.d.ts +1 -1
- package/dist/user-connect/connection-code.js +7 -7
- package/dist/user-connect/connection-code.test.js +106 -106
- package/dist/user-connect/index.d.ts +3 -3
- package/dist/user-connect/index.js +1 -1
- package/dist/user-connect/user-connect.pvars.js +13 -11
- package/dist/user-connect/user-connect.types.d.ts +3 -3
- package/dist/users.query.d.ts +2 -2
- package/dist/users.query.js +40 -30
- package/dist/utils.d.ts +2 -2
- package/dist/utils.js +34 -32
- package/dist/utils.test.js +12 -8
- package/dist/workflow-log-formatter.d.ts +1 -1
- package/dist/workflow-log-formatter.js +17 -18
- package/package.json +14 -8
|
@@ -9,10 +9,10 @@ class MockDataSource {
|
|
|
9
9
|
tableName;
|
|
10
10
|
primaryKeyName;
|
|
11
11
|
data = new Map();
|
|
12
|
-
constructor(tableName, primaryKeyName =
|
|
12
|
+
constructor(tableName, primaryKeyName = "fileId", initialData = []) {
|
|
13
13
|
this.tableName = tableName;
|
|
14
14
|
this.primaryKeyName = primaryKeyName;
|
|
15
|
-
initialData.forEach(item => {
|
|
15
|
+
initialData.forEach((item) => {
|
|
16
16
|
this.data.set(item[primaryKeyName], item);
|
|
17
17
|
});
|
|
18
18
|
}
|
|
@@ -35,7 +35,7 @@ class MockDataSource {
|
|
|
35
35
|
return record;
|
|
36
36
|
}
|
|
37
37
|
async delete(idOrRecord) {
|
|
38
|
-
const id = typeof idOrRecord ===
|
|
38
|
+
const id = typeof idOrRecord === "string" ? idOrRecord : idOrRecord[this.primaryKeyName];
|
|
39
39
|
this.data.delete(id);
|
|
40
40
|
}
|
|
41
41
|
async count() {
|
|
@@ -75,43 +75,43 @@ class MockFileOps {
|
|
|
75
75
|
return Array.from(this.files.keys());
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
|
-
describe(
|
|
78
|
+
describe("FileTable", () => {
|
|
79
79
|
let fileTable;
|
|
80
80
|
let mockFileOps;
|
|
81
81
|
beforeEach(() => {
|
|
82
82
|
// Create mock data source
|
|
83
|
-
const mockDataSource = new MockDataSource(
|
|
83
|
+
const mockDataSource = new MockDataSource("Files", "fileId");
|
|
84
84
|
// Use direct instantiation since factory requires setup
|
|
85
85
|
const metaData = {
|
|
86
|
-
name:
|
|
87
|
-
description:
|
|
88
|
-
primaryKeyName:
|
|
86
|
+
name: "Files",
|
|
87
|
+
description: "Files stored in the chunked file system for peer sharing",
|
|
88
|
+
primaryKeyName: "fileId",
|
|
89
89
|
fields: [
|
|
90
|
-
{ name:
|
|
91
|
-
{ name:
|
|
92
|
-
{ name:
|
|
93
|
-
{ name:
|
|
94
|
-
{ name:
|
|
95
|
-
{ name:
|
|
96
|
-
{ name:
|
|
97
|
-
{ name:
|
|
98
|
-
]
|
|
90
|
+
{ name: "fileId", type: field_type_1.FieldType.string },
|
|
91
|
+
{ name: "name", type: field_type_1.FieldType.string },
|
|
92
|
+
{ name: "fileSize", type: field_type_1.FieldType.number },
|
|
93
|
+
{ name: "fileHash", type: field_type_1.FieldType.string },
|
|
94
|
+
{ name: "mimeType", type: field_type_1.FieldType.string, optional: true },
|
|
95
|
+
{ name: "chunkHashes", type: field_type_1.FieldType.string, isArray: true, optional: true },
|
|
96
|
+
{ name: "isIndexFile", type: field_type_1.FieldType.boolean, optional: true },
|
|
97
|
+
{ name: "indexFileId", type: field_type_1.FieldType.string, optional: true },
|
|
98
|
+
],
|
|
99
99
|
};
|
|
100
100
|
// Create mock data context
|
|
101
101
|
const mockDataContextForRegistry = {
|
|
102
|
-
dataContextId:
|
|
102
|
+
dataContextId: "files-test",
|
|
103
103
|
};
|
|
104
|
-
const { EventRegistry } = require(
|
|
104
|
+
const { EventRegistry } = require("../orm/event-registry");
|
|
105
105
|
const eventRegistry = new EventRegistry(mockDataContextForRegistry);
|
|
106
|
-
const
|
|
106
|
+
const _mockDataContext = {
|
|
107
107
|
dataSourceFactory: () => mockDataSource,
|
|
108
|
-
eventRegistry: eventRegistry
|
|
108
|
+
eventRegistry: eventRegistry,
|
|
109
109
|
};
|
|
110
110
|
// Create FileTable instance directly with mock dataSource
|
|
111
111
|
const deps = {
|
|
112
112
|
dataSource: mockDataSource,
|
|
113
113
|
eventRegistry,
|
|
114
|
-
schema: file_types_1.fileSchema
|
|
114
|
+
schema: file_types_1.fileSchema,
|
|
115
115
|
};
|
|
116
116
|
fileTable = new files_1.FilesTable(metaData, deps);
|
|
117
117
|
// Set up mock file operations
|
|
@@ -122,41 +122,41 @@ describe('FileTable', () => {
|
|
|
122
122
|
// Clean up global state to prevent Jest from hanging
|
|
123
123
|
(0, file_types_1.resetFileOps)();
|
|
124
124
|
});
|
|
125
|
-
describe(
|
|
126
|
-
it(
|
|
125
|
+
describe("saveFile", () => {
|
|
126
|
+
it("should save a small file in a single chunk", async () => {
|
|
127
127
|
const fileId = (0, utils_1.newid)();
|
|
128
|
-
const data = new Uint8Array(Buffer.from(
|
|
128
|
+
const data = new Uint8Array(Buffer.from("Hello, World!", "utf8"));
|
|
129
129
|
const metadata = {
|
|
130
130
|
fileId,
|
|
131
|
-
name:
|
|
131
|
+
name: "test.txt",
|
|
132
132
|
fileSize: data.length,
|
|
133
|
-
mimeType:
|
|
133
|
+
mimeType: "text/plain",
|
|
134
134
|
};
|
|
135
135
|
const result = await fileTable.saveFile(metadata, data);
|
|
136
136
|
expect(result.fileId).toBe(fileId);
|
|
137
137
|
expect(result.chunkHashes).toHaveLength(1);
|
|
138
138
|
// Check that chunk was written by hash
|
|
139
|
-
const chunkHash = result.chunkHashes[0];
|
|
139
|
+
const chunkHash = result.chunkHashes?.[0];
|
|
140
140
|
const chunkPath = `file_chunks/${chunkHash}`;
|
|
141
141
|
expect(await mockFileOps.fileExists(chunkPath)).toBe(true);
|
|
142
142
|
const storedChunk = await mockFileOps.readFile(chunkPath);
|
|
143
143
|
expect(storedChunk).toBeInstanceOf(Uint8Array);
|
|
144
144
|
expect(storedChunk).toEqual(data);
|
|
145
145
|
});
|
|
146
|
-
it(
|
|
146
|
+
it("should save a large file in multiple chunks", async () => {
|
|
147
147
|
const fileId = (0, utils_1.newid)();
|
|
148
|
-
const largeData = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE + 1000,
|
|
148
|
+
const largeData = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE + 1000, "A")); // Slightly larger than one chunk
|
|
149
149
|
const metadata = {
|
|
150
150
|
fileId,
|
|
151
|
-
name:
|
|
151
|
+
name: "large.txt",
|
|
152
152
|
fileSize: largeData.length,
|
|
153
|
-
mimeType:
|
|
153
|
+
mimeType: "text/plain",
|
|
154
154
|
};
|
|
155
155
|
const result = await fileTable.saveFile(metadata, largeData);
|
|
156
156
|
expect(result.chunkHashes).toHaveLength(2);
|
|
157
157
|
// Check that both chunks were written by hash
|
|
158
|
-
const chunk0Hash = result.chunkHashes[0];
|
|
159
|
-
const chunk1Hash = result.chunkHashes[1];
|
|
158
|
+
const chunk0Hash = result.chunkHashes?.[0];
|
|
159
|
+
const chunk1Hash = result.chunkHashes?.[1];
|
|
160
160
|
expect(await mockFileOps.fileExists(`file_chunks/${chunk0Hash}`)).toBe(true);
|
|
161
161
|
expect(await mockFileOps.fileExists(`file_chunks/${chunk1Hash}`)).toBe(true);
|
|
162
162
|
// Verify chunk sizes
|
|
@@ -166,15 +166,15 @@ describe('FileTable', () => {
|
|
|
166
166
|
expect(chunk1.length).toBe(1000);
|
|
167
167
|
});
|
|
168
168
|
});
|
|
169
|
-
describe(
|
|
170
|
-
it(
|
|
169
|
+
describe("getFile", () => {
|
|
170
|
+
it("should retrieve a single-chunk file", async () => {
|
|
171
171
|
const fileId = (0, utils_1.newid)();
|
|
172
|
-
const originalData = new Uint8Array(Buffer.from(
|
|
172
|
+
const originalData = new Uint8Array(Buffer.from("Test content", "utf8"));
|
|
173
173
|
const metadata = {
|
|
174
174
|
fileId,
|
|
175
|
-
name:
|
|
175
|
+
name: "test.txt",
|
|
176
176
|
fileSize: originalData.length,
|
|
177
|
-
mimeType:
|
|
177
|
+
mimeType: "text/plain",
|
|
178
178
|
};
|
|
179
179
|
// Save file first
|
|
180
180
|
await fileTable.saveFile(metadata, originalData);
|
|
@@ -182,14 +182,14 @@ describe('FileTable', () => {
|
|
|
182
182
|
const retrievedData = await fileTable.getFileContents(fileId);
|
|
183
183
|
expect(new Uint8Array(retrievedData)).toEqual(originalData);
|
|
184
184
|
});
|
|
185
|
-
it(
|
|
185
|
+
it("should retrieve a multi-chunk file", async () => {
|
|
186
186
|
const fileId = (0, utils_1.newid)();
|
|
187
|
-
const originalData = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE + 500,
|
|
187
|
+
const originalData = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE + 500, "B"));
|
|
188
188
|
const metadata = {
|
|
189
189
|
fileId,
|
|
190
|
-
name:
|
|
190
|
+
name: "large.txt",
|
|
191
191
|
fileSize: originalData.length,
|
|
192
|
-
mimeType:
|
|
192
|
+
mimeType: "text/plain",
|
|
193
193
|
};
|
|
194
194
|
// Save file first
|
|
195
195
|
await fileTable.saveFile(metadata, originalData);
|
|
@@ -197,45 +197,45 @@ describe('FileTable', () => {
|
|
|
197
197
|
const retrievedData = await fileTable.getFileContents(fileId);
|
|
198
198
|
expect(new Uint8Array(retrievedData)).toEqual(originalData);
|
|
199
199
|
});
|
|
200
|
-
it(
|
|
201
|
-
const result = await fileTable.getFileContents(
|
|
200
|
+
it("should return null for non-existent file", async () => {
|
|
201
|
+
const result = await fileTable.getFileContents("non-existent");
|
|
202
202
|
expect(result).toBeNull();
|
|
203
203
|
});
|
|
204
|
-
it(
|
|
204
|
+
it("should return null when chunk is missing", async () => {
|
|
205
205
|
const fileId = (0, utils_1.newid)();
|
|
206
206
|
const metadata = {
|
|
207
207
|
fileId,
|
|
208
|
-
name:
|
|
208
|
+
name: "test.txt",
|
|
209
209
|
fileSize: 100,
|
|
210
|
-
fileHash:
|
|
211
|
-
chunkHashes: [
|
|
210
|
+
fileHash: "hash123",
|
|
211
|
+
chunkHashes: ["hash1", "hash2"],
|
|
212
212
|
};
|
|
213
213
|
// Insert metadata directly without saving chunks
|
|
214
214
|
await fileTable.dataSource.insert(metadata);
|
|
215
215
|
const result = await fileTable.getFileContents(fileId);
|
|
216
216
|
expect(result).toBeNull();
|
|
217
217
|
});
|
|
218
|
-
it(
|
|
219
|
-
const result = await fileTable.getFileContents(
|
|
218
|
+
it("should return null when chunk is missing during read", async () => {
|
|
219
|
+
const result = await fileTable.getFileContents("non-existent");
|
|
220
220
|
expect(result).toBeNull();
|
|
221
221
|
});
|
|
222
222
|
});
|
|
223
|
-
describe(
|
|
224
|
-
it(
|
|
223
|
+
describe("deleteFile", () => {
|
|
224
|
+
it("should delete file from database but preserve chunks", async () => {
|
|
225
225
|
const fileId = (0, utils_1.newid)();
|
|
226
|
-
const data = new Uint8Array(Buffer.from(
|
|
226
|
+
const data = new Uint8Array(Buffer.from("Delete me", "utf8"));
|
|
227
227
|
const metadata = {
|
|
228
228
|
fileId,
|
|
229
|
-
name:
|
|
229
|
+
name: "delete.txt",
|
|
230
230
|
fileSize: data.length,
|
|
231
|
-
mimeType:
|
|
231
|
+
mimeType: "text/plain",
|
|
232
232
|
};
|
|
233
233
|
// Save file first
|
|
234
234
|
const saved = await fileTable.saveFile(metadata, data);
|
|
235
235
|
// Verify file exists
|
|
236
236
|
const fileData = await fileTable.getFileContents(fileId);
|
|
237
237
|
expect(new Uint8Array(fileData)).toEqual(data);
|
|
238
|
-
const chunkHash = saved.chunkHashes[0];
|
|
238
|
+
const chunkHash = saved.chunkHashes?.[0];
|
|
239
239
|
expect(await mockFileOps.fileExists(`file_chunks/${chunkHash}`)).toBe(true);
|
|
240
240
|
// Delete file
|
|
241
241
|
await fileTable.deleteFile(fileId);
|
|
@@ -243,24 +243,24 @@ describe('FileTable', () => {
|
|
|
243
243
|
expect(await fileTable.getFileContents(fileId)).toBeNull();
|
|
244
244
|
expect(await mockFileOps.fileExists(`file_chunks/${chunkHash}`)).toBe(true);
|
|
245
245
|
});
|
|
246
|
-
it(
|
|
246
|
+
it("should handle deleting non-existent file gracefully", async () => {
|
|
247
247
|
// Should not throw
|
|
248
|
-
await fileTable.deleteFile(
|
|
248
|
+
await fileTable.deleteFile("non-existent");
|
|
249
249
|
});
|
|
250
|
-
it(
|
|
250
|
+
it("should delete multi-chunk file from database but preserve chunks", async () => {
|
|
251
251
|
const fileId = (0, utils_1.newid)();
|
|
252
|
-
const data = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE + 100,
|
|
252
|
+
const data = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE + 100, "C"));
|
|
253
253
|
const metadata = {
|
|
254
254
|
fileId,
|
|
255
|
-
name:
|
|
255
|
+
name: "multi.txt",
|
|
256
256
|
fileSize: data.length,
|
|
257
|
-
mimeType:
|
|
257
|
+
mimeType: "text/plain",
|
|
258
258
|
};
|
|
259
259
|
// Save file first
|
|
260
260
|
const saved = await fileTable.saveFile(metadata, data);
|
|
261
261
|
// Verify chunks exist by hash
|
|
262
|
-
const chunk0Hash = saved.chunkHashes[0];
|
|
263
|
-
const chunk1Hash = saved.chunkHashes[1];
|
|
262
|
+
const chunk0Hash = saved.chunkHashes?.[0];
|
|
263
|
+
const chunk1Hash = saved.chunkHashes?.[1];
|
|
264
264
|
expect(await mockFileOps.fileExists(`file_chunks/${chunk0Hash}`)).toBe(true);
|
|
265
265
|
expect(await mockFileOps.fileExists(`file_chunks/${chunk1Hash}`)).toBe(true);
|
|
266
266
|
// Delete file
|
|
@@ -271,23 +271,23 @@ describe('FileTable', () => {
|
|
|
271
271
|
expect(await mockFileOps.fileExists(`file_chunks/${chunk1Hash}`)).toBe(true);
|
|
272
272
|
});
|
|
273
273
|
});
|
|
274
|
-
describe(
|
|
275
|
-
it(
|
|
274
|
+
describe("chunk deduplication", () => {
|
|
275
|
+
it("should deduplicate identical chunks across different files", async () => {
|
|
276
276
|
const fileId1 = (0, utils_1.newid)();
|
|
277
277
|
const fileId2 = (0, utils_1.newid)();
|
|
278
278
|
// Create two files with identical content
|
|
279
|
-
const data = new Uint8Array(Buffer.from(
|
|
279
|
+
const data = new Uint8Array(Buffer.from("Identical content for deduplication test", "utf8"));
|
|
280
280
|
const metadata1 = {
|
|
281
281
|
fileId: fileId1,
|
|
282
|
-
name:
|
|
282
|
+
name: "file1.txt",
|
|
283
283
|
fileSize: data.length,
|
|
284
|
-
mimeType:
|
|
284
|
+
mimeType: "text/plain",
|
|
285
285
|
};
|
|
286
286
|
const metadata2 = {
|
|
287
287
|
fileId: fileId2,
|
|
288
|
-
name:
|
|
288
|
+
name: "file2.txt",
|
|
289
289
|
fileSize: data.length,
|
|
290
|
-
mimeType:
|
|
290
|
+
mimeType: "text/plain",
|
|
291
291
|
};
|
|
292
292
|
// Save both files
|
|
293
293
|
const saved1 = await fileTable.saveFile(metadata1, data);
|
|
@@ -295,10 +295,11 @@ describe('FileTable', () => {
|
|
|
295
295
|
// Both files should have the same chunk hash (deduplication)
|
|
296
296
|
expect(saved1.chunkHashes).toEqual(saved2.chunkHashes);
|
|
297
297
|
// Only one chunk should be stored physically
|
|
298
|
+
expect(saved1.chunkHashes?.length).toBeGreaterThan(0);
|
|
298
299
|
const chunkHash = saved1.chunkHashes[0];
|
|
299
300
|
const chunkPath = `file_chunks/${chunkHash}`;
|
|
300
301
|
expect(await mockFileOps.fileExists(chunkPath)).toBe(true);
|
|
301
|
-
expect(mockFileOps.getStoredFiles().filter(path => path.includes(chunkHash))).toHaveLength(1);
|
|
302
|
+
expect(mockFileOps.getStoredFiles().filter((path) => path.includes(chunkHash))).toHaveLength(1);
|
|
302
303
|
// Both files should retrieve the same content
|
|
303
304
|
const file1Data = await fileTable.getFileContents(fileId1);
|
|
304
305
|
const file2Data = await fileTable.getFileContents(fileId2);
|
|
@@ -306,16 +307,16 @@ describe('FileTable', () => {
|
|
|
306
307
|
expect(new Uint8Array(file2Data)).toEqual(data);
|
|
307
308
|
});
|
|
308
309
|
});
|
|
309
|
-
describe(
|
|
310
|
-
it(
|
|
310
|
+
describe("Merkle tree (chunk index files)", () => {
|
|
311
|
+
it("should use direct chunk hashes for files under threshold", async () => {
|
|
311
312
|
const fileId = (0, utils_1.newid)();
|
|
312
313
|
// Create a small file that's well under the 1000 chunk threshold
|
|
313
|
-
const smallData = new Uint8Array(Buffer.from(
|
|
314
|
+
const smallData = new Uint8Array(Buffer.from("Small file content for testing", "utf8"));
|
|
314
315
|
const metadata = {
|
|
315
316
|
fileId,
|
|
316
|
-
name:
|
|
317
|
+
name: "small.bin",
|
|
317
318
|
fileSize: smallData.length,
|
|
318
|
-
mimeType:
|
|
319
|
+
mimeType: "application/octet-stream",
|
|
319
320
|
};
|
|
320
321
|
// Save small file
|
|
321
322
|
const result = await fileTable.saveFile(metadata, smallData);
|
|
@@ -327,19 +328,19 @@ describe('FileTable', () => {
|
|
|
327
328
|
const retrievedData = await fileTable.getFileContents(fileId);
|
|
328
329
|
expect(new Uint8Array(retrievedData)).toEqual(smallData);
|
|
329
330
|
});
|
|
330
|
-
it(
|
|
331
|
+
it("should use recursive index files for very large files", async () => {
|
|
331
332
|
// Temporarily lower the threshold to test recursive behavior with smaller data
|
|
332
333
|
const originalThreshold = file_types_1.CHUNK_INDEX_THRESHOLD;
|
|
333
334
|
(0, file_types_1.setChunkIndexThreshold)(3); // Use index file for files with >3 chunks
|
|
334
335
|
try {
|
|
335
336
|
const fileId = (0, utils_1.newid)();
|
|
336
337
|
// Create a file with 5 chunks (exceeds threshold of 3)
|
|
337
|
-
const largeData = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE * 4 + 1000,
|
|
338
|
+
const largeData = new Uint8Array(Buffer.alloc(file_types_1.FILE_CHUNK_SIZE * 4 + 1000, "X")); // 4 full chunks + partial chunk = 5 chunks
|
|
338
339
|
const metadata = {
|
|
339
340
|
fileId,
|
|
340
|
-
name:
|
|
341
|
+
name: "large.bin",
|
|
341
342
|
fileSize: largeData.length,
|
|
342
|
-
mimeType:
|
|
343
|
+
mimeType: "application/octet-stream",
|
|
343
344
|
};
|
|
344
345
|
// Save large file - should create recursive index file
|
|
345
346
|
const result = await fileTable.saveFile(metadata, largeData);
|
|
@@ -353,68 +354,68 @@ describe('FileTable', () => {
|
|
|
353
354
|
// The index file should be marked as an index file
|
|
354
355
|
const indexFile = await fileTable.dataSource.get(result.indexFileId);
|
|
355
356
|
expect(indexFile).toBeTruthy();
|
|
356
|
-
expect(indexFile
|
|
357
|
+
expect(indexFile?.isIndexFile).toBe(true);
|
|
357
358
|
// The index file should contain JSON data
|
|
358
|
-
expect(indexFile
|
|
359
|
-
expect(indexFile
|
|
359
|
+
expect(indexFile?.mimeType).toBe("application/json");
|
|
360
|
+
expect(indexFile?.name).toMatch(/^index-.*\.json$/);
|
|
360
361
|
}
|
|
361
362
|
finally {
|
|
362
363
|
// Restore original threshold
|
|
363
364
|
(0, file_types_1.setChunkIndexThreshold)(originalThreshold);
|
|
364
365
|
}
|
|
365
366
|
});
|
|
366
|
-
it(
|
|
367
|
+
it("should handle missing index file gracefully", async () => {
|
|
367
368
|
const fileId = (0, utils_1.newid)();
|
|
368
369
|
const metadata = {
|
|
369
370
|
fileId,
|
|
370
|
-
name:
|
|
371
|
+
name: "test.txt",
|
|
371
372
|
fileSize: 100,
|
|
372
|
-
fileHash:
|
|
373
|
-
indexFileId:
|
|
373
|
+
fileHash: "hash123",
|
|
374
|
+
indexFileId: "missing_index_file_id",
|
|
374
375
|
};
|
|
375
376
|
// Insert metadata directly without creating index file
|
|
376
377
|
await fileTable.dataSource.insert(metadata);
|
|
377
|
-
await expect(fileTable.getFileContents(fileId)).rejects.toThrow(
|
|
378
|
+
await expect(fileTable.getFileContents(fileId)).rejects.toThrow("Index file not found: missing_index_file_id");
|
|
378
379
|
});
|
|
379
380
|
});
|
|
380
|
-
describe(
|
|
381
|
-
describe(
|
|
382
|
-
it(
|
|
381
|
+
describe("Streaming API", () => {
|
|
382
|
+
describe("FileWriteStream", () => {
|
|
383
|
+
it("should create a write stream and write small file chunk by chunk", async () => {
|
|
383
384
|
const fileId = (0, utils_1.newid)();
|
|
384
385
|
const metadata = {
|
|
385
386
|
fileId,
|
|
386
|
-
name:
|
|
387
|
+
name: "stream-test.txt",
|
|
387
388
|
fileSize: 0, // Will be calculated
|
|
388
|
-
mimeType:
|
|
389
|
+
mimeType: "text/plain",
|
|
389
390
|
};
|
|
390
391
|
const writeStream = await fileTable.createWriteStream(metadata);
|
|
391
392
|
// Write data in chunks
|
|
392
|
-
await writeStream.write(new Uint8Array(Buffer.from(
|
|
393
|
-
await writeStream.write(new Uint8Array(Buffer.from(
|
|
394
|
-
await writeStream.write(new Uint8Array(Buffer.from(
|
|
393
|
+
await writeStream.write(new Uint8Array(Buffer.from("Hello, ", "utf8")));
|
|
394
|
+
await writeStream.write(new Uint8Array(Buffer.from("streaming ", "utf8")));
|
|
395
|
+
await writeStream.write(new Uint8Array(Buffer.from("world!", "utf8")));
|
|
395
396
|
// Finalize the stream
|
|
396
397
|
const result = await writeStream.finalize();
|
|
397
398
|
expect(result.fileId).toBe(fileId);
|
|
398
|
-
expect(result.name).toBe(
|
|
399
|
+
expect(result.name).toBe("stream-test.txt");
|
|
399
400
|
expect(result.fileSize).toBe(23); // Total bytes written: 'Hello, streaming world!'
|
|
400
401
|
expect(result.chunkHashes).toHaveLength(1); // Small file, single chunk
|
|
401
402
|
// Verify we can read the file back
|
|
402
403
|
const retrievedData = await fileTable.getFileContents(fileId);
|
|
403
|
-
expect(Buffer.from(retrievedData).toString(
|
|
404
|
+
expect(Buffer.from(retrievedData).toString("utf8")).toBe("Hello, streaming world!");
|
|
404
405
|
});
|
|
405
|
-
it(
|
|
406
|
+
it("should handle large file streaming across multiple chunks", async () => {
|
|
406
407
|
const fileId = (0, utils_1.newid)();
|
|
407
408
|
const metadata = {
|
|
408
409
|
fileId,
|
|
409
|
-
name:
|
|
410
|
+
name: "large-stream.bin",
|
|
410
411
|
fileSize: 0,
|
|
411
|
-
mimeType:
|
|
412
|
+
mimeType: "application/octet-stream",
|
|
412
413
|
};
|
|
413
414
|
const writeStream = await fileTable.createWriteStream(metadata);
|
|
414
415
|
// Write data larger than one chunk
|
|
415
416
|
const chunkSize = file_types_1.FILE_CHUNK_SIZE;
|
|
416
|
-
const chunk1 = new Uint8Array(Buffer.alloc(chunkSize,
|
|
417
|
-
const chunk2 = new Uint8Array(Buffer.alloc(500,
|
|
417
|
+
const chunk1 = new Uint8Array(Buffer.alloc(chunkSize, "A"));
|
|
418
|
+
const chunk2 = new Uint8Array(Buffer.alloc(500, "B")); // Partial second chunk
|
|
418
419
|
await writeStream.write(chunk1);
|
|
419
420
|
await writeStream.write(chunk2);
|
|
420
421
|
const result = await writeStream.finalize();
|
|
@@ -423,56 +424,54 @@ describe('FileTable', () => {
|
|
|
423
424
|
// Verify content
|
|
424
425
|
const retrievedData = await fileTable.getFileContents(fileId);
|
|
425
426
|
expect(retrievedData?.length).toBe(chunkSize + 500);
|
|
426
|
-
expect(new Uint8Array(retrievedData).subarray(0, chunkSize).every(b => b === 65)).toBe(true); // All 'A's
|
|
427
|
-
expect(new Uint8Array(retrievedData).subarray(chunkSize).every(b => b === 66)).toBe(true); // All 'B's
|
|
427
|
+
expect(new Uint8Array(retrievedData).subarray(0, chunkSize).every((b) => b === 65)).toBe(true); // All 'A's
|
|
428
|
+
expect(new Uint8Array(retrievedData).subarray(chunkSize).every((b) => b === 66)).toBe(true); // All 'B's
|
|
428
429
|
});
|
|
429
|
-
it(
|
|
430
|
+
it("should support aborting a write stream", async () => {
|
|
430
431
|
const fileId = (0, utils_1.newid)();
|
|
431
432
|
const metadata = {
|
|
432
433
|
fileId,
|
|
433
|
-
name:
|
|
434
|
+
name: "aborted.txt",
|
|
434
435
|
fileSize: 0,
|
|
435
|
-
mimeType:
|
|
436
|
+
mimeType: "text/plain",
|
|
436
437
|
};
|
|
437
438
|
const writeStream = await fileTable.createWriteStream(metadata);
|
|
438
|
-
await writeStream.write(new Uint8Array(Buffer.from(
|
|
439
|
+
await writeStream.write(new Uint8Array(Buffer.from("Some data", "utf8")));
|
|
439
440
|
expect(writeStream.getBytesWritten()).toBe(9);
|
|
440
441
|
// Abort the stream
|
|
441
442
|
await writeStream.abort();
|
|
442
443
|
expect(writeStream.isAborted()).toBe(true);
|
|
443
444
|
// Should not be able to write after abort
|
|
444
|
-
await expect(writeStream.write(new Uint8Array(Buffer.from(
|
|
445
|
-
.rejects.toThrow('Cannot write to aborted stream');
|
|
445
|
+
await expect(writeStream.write(new Uint8Array(Buffer.from("More data", "utf8")))).rejects.toThrow("Cannot write to aborted stream");
|
|
446
446
|
// Should not be able to finalize after abort
|
|
447
|
-
await expect(writeStream.finalize())
|
|
448
|
-
.rejects.toThrow('Cannot finalize aborted stream');
|
|
447
|
+
await expect(writeStream.finalize()).rejects.toThrow("Cannot finalize aborted stream");
|
|
449
448
|
});
|
|
450
|
-
it(
|
|
449
|
+
it("should track progress during streaming", async () => {
|
|
451
450
|
const fileId = (0, utils_1.newid)();
|
|
452
451
|
const metadata = {
|
|
453
452
|
fileId,
|
|
454
|
-
name:
|
|
453
|
+
name: "progress.txt",
|
|
455
454
|
fileSize: 0,
|
|
456
|
-
mimeType:
|
|
455
|
+
mimeType: "text/plain",
|
|
457
456
|
};
|
|
458
457
|
const writeStream = await fileTable.createWriteStream(metadata);
|
|
459
458
|
expect(writeStream.getBytesWritten()).toBe(0);
|
|
460
459
|
expect(writeStream.getChunkCount()).toBe(0);
|
|
461
|
-
await writeStream.write(new Uint8Array(Buffer.from(
|
|
460
|
+
await writeStream.write(new Uint8Array(Buffer.from("First chunk", "utf8")));
|
|
462
461
|
expect(writeStream.getBytesWritten()).toBe(11);
|
|
463
462
|
expect(writeStream.getChunkCount()).toBe(0); // Not complete chunk yet
|
|
464
|
-
await writeStream.write(new Uint8Array(Buffer.from(
|
|
463
|
+
await writeStream.write(new Uint8Array(Buffer.from(" Second part", "utf8")));
|
|
465
464
|
expect(writeStream.getBytesWritten()).toBe(23);
|
|
466
465
|
await writeStream.finalize();
|
|
467
466
|
expect(writeStream.isFinalized()).toBe(true);
|
|
468
467
|
});
|
|
469
|
-
it(
|
|
468
|
+
it("should handle finalize with multiple complete chunks in buffer", async () => {
|
|
470
469
|
const fileId = (0, utils_1.newid)();
|
|
471
470
|
const metadata = {
|
|
472
471
|
fileId,
|
|
473
|
-
name:
|
|
472
|
+
name: "buffered-chunks.bin",
|
|
474
473
|
fileSize: 0,
|
|
475
|
-
mimeType:
|
|
474
|
+
mimeType: "application/octet-stream",
|
|
476
475
|
};
|
|
477
476
|
const writeStream = await fileTable.createWriteStream(metadata);
|
|
478
477
|
// Write data that will create multiple complete chunks but don't trigger processCompleteChunks
|
|
@@ -495,122 +494,122 @@ describe('FileTable', () => {
|
|
|
495
494
|
expect(new Uint8Array(retrievedData)).toEqual(data);
|
|
496
495
|
});
|
|
497
496
|
});
|
|
498
|
-
describe(
|
|
499
|
-
it(
|
|
497
|
+
describe("FileReadStream", () => {
|
|
498
|
+
it("should read file data in chunks using read stream", async () => {
|
|
500
499
|
// First save a file using traditional method
|
|
501
500
|
const fileId = (0, utils_1.newid)();
|
|
502
|
-
const originalData = new Uint8Array(Buffer.from(
|
|
501
|
+
const originalData = new Uint8Array(Buffer.from("This is test data for streaming read operations!", "utf8"));
|
|
503
502
|
const metadata = {
|
|
504
503
|
fileId,
|
|
505
|
-
name:
|
|
504
|
+
name: "read-test.txt",
|
|
506
505
|
fileSize: originalData.length,
|
|
507
|
-
mimeType:
|
|
506
|
+
mimeType: "text/plain",
|
|
508
507
|
};
|
|
509
508
|
await fileTable.saveFile(metadata, originalData);
|
|
510
509
|
// Now open for streaming read
|
|
511
510
|
const readStream = await fileTable.openReadStream(fileId);
|
|
512
511
|
expect(readStream).toBeTruthy();
|
|
513
512
|
// Read entire file
|
|
514
|
-
const readData = await readStream
|
|
513
|
+
const readData = await readStream?.readAll();
|
|
515
514
|
expect(readData).not.toBeNull();
|
|
516
515
|
expect(new Uint8Array(readData)).toEqual(originalData);
|
|
517
516
|
// Check metadata
|
|
518
517
|
const meta = readStream.getMetadata();
|
|
519
518
|
expect(meta.fileId).toBe(fileId);
|
|
520
|
-
expect(meta.name).toBe(
|
|
519
|
+
expect(meta.name).toBe("read-test.txt");
|
|
521
520
|
expect(meta.fileSize).toBe(originalData.length);
|
|
522
521
|
});
|
|
523
|
-
it(
|
|
522
|
+
it("should support seeking within file", async () => {
|
|
524
523
|
const fileId = (0, utils_1.newid)();
|
|
525
|
-
const originalData = new Uint8Array(Buffer.from(
|
|
524
|
+
const originalData = new Uint8Array(Buffer.from("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ", "utf8"));
|
|
526
525
|
const metadata = {
|
|
527
526
|
fileId,
|
|
528
|
-
name:
|
|
527
|
+
name: "seek-test.txt",
|
|
529
528
|
fileSize: originalData.length,
|
|
530
|
-
mimeType:
|
|
529
|
+
mimeType: "text/plain",
|
|
531
530
|
};
|
|
532
531
|
await fileTable.saveFile(metadata, originalData);
|
|
533
532
|
const readStream = await fileTable.openReadStream(fileId);
|
|
534
533
|
expect(readStream).toBeTruthy();
|
|
535
534
|
// Seek to position 10 (should be at 'A')
|
|
536
|
-
await readStream
|
|
537
|
-
expect(readStream
|
|
535
|
+
await readStream?.seek(10);
|
|
536
|
+
expect(readStream?.getPosition()).toBe(10);
|
|
538
537
|
// Read 5 bytes from position 10 (should get 'ABCDE')
|
|
539
|
-
const chunk = await readStream
|
|
540
|
-
expect(Buffer.from(chunk).toString(
|
|
541
|
-
expect(readStream
|
|
538
|
+
const chunk = await readStream?.read(5);
|
|
539
|
+
expect(Buffer.from(chunk).toString("utf8")).toBe("ABCDE");
|
|
540
|
+
expect(readStream?.getPosition()).toBe(15);
|
|
542
541
|
// Seek back to beginning
|
|
543
|
-
await readStream
|
|
544
|
-
expect(readStream
|
|
545
|
-
const firstChunk = await readStream
|
|
546
|
-
expect(Buffer.from(firstChunk).toString(
|
|
547
|
-
expect(readStream
|
|
542
|
+
await readStream?.seek(0);
|
|
543
|
+
expect(readStream?.getPosition()).toBe(0);
|
|
544
|
+
const firstChunk = await readStream?.read(10);
|
|
545
|
+
expect(Buffer.from(firstChunk).toString("utf8")).toBe("0123456789");
|
|
546
|
+
expect(readStream?.getPosition()).toBe(10);
|
|
548
547
|
// Seek to middle of data
|
|
549
|
-
await readStream
|
|
550
|
-
expect(readStream
|
|
551
|
-
const middleChunk = await readStream
|
|
552
|
-
expect(Buffer.from(middleChunk).toString(
|
|
553
|
-
expect(readStream
|
|
548
|
+
await readStream?.seek(20);
|
|
549
|
+
expect(readStream?.getPosition()).toBe(20);
|
|
550
|
+
const middleChunk = await readStream?.read(5);
|
|
551
|
+
expect(Buffer.from(middleChunk).toString("utf8")).toBe("KLMNO");
|
|
552
|
+
expect(readStream?.getPosition()).toBe(25);
|
|
554
553
|
});
|
|
555
|
-
it(
|
|
554
|
+
it("should handle seeking across chunk boundaries in large files", async () => {
|
|
556
555
|
// Create a file that spans multiple chunks
|
|
557
556
|
const fileId = (0, utils_1.newid)();
|
|
558
557
|
const chunkSize = file_types_1.FILE_CHUNK_SIZE;
|
|
559
|
-
const data1 = Buffer.alloc(chunkSize,
|
|
560
|
-
const data2 = Buffer.alloc(chunkSize,
|
|
561
|
-
const data3 = Buffer.alloc(500,
|
|
558
|
+
const data1 = Buffer.alloc(chunkSize, "A"); // First chunk: all A's
|
|
559
|
+
const data2 = Buffer.alloc(chunkSize, "B"); // Second chunk: all B's
|
|
560
|
+
const data3 = Buffer.alloc(500, "C"); // Third partial chunk: all C's
|
|
562
561
|
const largeData = new Uint8Array(Buffer.concat([data1, data2, data3]));
|
|
563
562
|
const metadata = {
|
|
564
563
|
fileId,
|
|
565
|
-
name:
|
|
564
|
+
name: "large-seek-test.bin",
|
|
566
565
|
fileSize: largeData.length,
|
|
567
|
-
mimeType:
|
|
566
|
+
mimeType: "application/octet-stream",
|
|
568
567
|
};
|
|
569
568
|
await fileTable.saveFile(metadata, largeData);
|
|
570
569
|
const readStream = await fileTable.openReadStream(fileId);
|
|
571
570
|
expect(readStream).toBeTruthy();
|
|
572
571
|
// Seek to end of first chunk
|
|
573
|
-
await readStream
|
|
574
|
-
const endOfFirst = await readStream
|
|
572
|
+
await readStream?.seek(chunkSize - 5);
|
|
573
|
+
const endOfFirst = await readStream?.read(10); // Should span chunk boundary
|
|
575
574
|
expect(endOfFirst?.length).toBe(10);
|
|
576
|
-
expect(endOfFirst?.subarray(0, 5).every(b => b === 65)).toBe(true); // First 5 bytes: A's
|
|
577
|
-
expect(endOfFirst?.subarray(5, 10).every(b => b === 66)).toBe(true); // Next 5 bytes: B's
|
|
575
|
+
expect(endOfFirst?.subarray(0, 5).every((b) => b === 65)).toBe(true); // First 5 bytes: A's
|
|
576
|
+
expect(endOfFirst?.subarray(5, 10).every((b) => b === 66)).toBe(true); // Next 5 bytes: B's
|
|
578
577
|
// Seek to middle of second chunk
|
|
579
|
-
await readStream
|
|
580
|
-
const middleOfSecond = await readStream
|
|
578
|
+
await readStream?.seek(chunkSize + 1000);
|
|
579
|
+
const middleOfSecond = await readStream?.read(100);
|
|
581
580
|
expect(middleOfSecond?.length).toBe(100);
|
|
582
|
-
expect(middleOfSecond?.every(b => b === 66)).toBe(true); // All B's
|
|
581
|
+
expect(middleOfSecond?.every((b) => b === 66)).toBe(true); // All B's
|
|
583
582
|
// Seek to third chunk
|
|
584
|
-
await readStream
|
|
585
|
-
const inThirdChunk = await readStream
|
|
583
|
+
await readStream?.seek(chunkSize * 2 + 100);
|
|
584
|
+
const inThirdChunk = await readStream?.read(200);
|
|
586
585
|
expect(inThirdChunk?.length).toBe(200);
|
|
587
|
-
expect(inThirdChunk?.every(b => b === 67)).toBe(true); // All C's
|
|
586
|
+
expect(inThirdChunk?.every((b) => b === 67)).toBe(true); // All C's
|
|
588
587
|
});
|
|
589
|
-
it(
|
|
588
|
+
it("should handle seek error conditions", async () => {
|
|
590
589
|
const fileId = (0, utils_1.newid)();
|
|
591
|
-
const data = new Uint8Array(Buffer.from(
|
|
590
|
+
const data = new Uint8Array(Buffer.from("Short test data", "utf8"));
|
|
592
591
|
const metadata = {
|
|
593
592
|
fileId,
|
|
594
|
-
name:
|
|
593
|
+
name: "error-test.txt",
|
|
595
594
|
fileSize: data.length,
|
|
596
|
-
mimeType:
|
|
595
|
+
mimeType: "text/plain",
|
|
597
596
|
};
|
|
598
597
|
await fileTable.saveFile(metadata, data);
|
|
599
598
|
const readStream = await fileTable.openReadStream(fileId);
|
|
600
599
|
expect(readStream).toBeTruthy();
|
|
601
600
|
// Test negative seek position
|
|
602
|
-
await expect(readStream
|
|
601
|
+
await expect(readStream?.seek(-1)).rejects.toThrow("Seek position cannot be negative");
|
|
603
602
|
// Test seeking beyond file size
|
|
604
|
-
await expect(readStream
|
|
603
|
+
await expect(readStream?.seek(data.length + 1)).rejects.toThrow("Seek position beyond file size");
|
|
605
604
|
// Test seeking to exact end of file
|
|
606
|
-
await readStream
|
|
607
|
-
expect(readStream
|
|
608
|
-
expect(readStream
|
|
605
|
+
await readStream?.seek(data.length);
|
|
606
|
+
expect(readStream?.getPosition()).toBe(data.length);
|
|
607
|
+
expect(readStream?.getBytesRemaining()).toBe(0);
|
|
609
608
|
// Should return null when reading at EOF
|
|
610
|
-
const atEof = await readStream
|
|
609
|
+
const atEof = await readStream?.read();
|
|
611
610
|
expect(atEof).toBeNull();
|
|
612
611
|
});
|
|
613
|
-
it(
|
|
612
|
+
it("should handle large file streaming reads", async () => {
|
|
614
613
|
// Create a large file with known pattern
|
|
615
614
|
const fileId = (0, utils_1.newid)();
|
|
616
615
|
const chunkSize = file_types_1.FILE_CHUNK_SIZE;
|
|
@@ -621,66 +620,66 @@ describe('FileTable', () => {
|
|
|
621
620
|
const largeData = new Uint8Array(buffer);
|
|
622
621
|
const metadata = {
|
|
623
622
|
fileId,
|
|
624
|
-
name:
|
|
623
|
+
name: "large-read.bin",
|
|
625
624
|
fileSize: largeData.length,
|
|
626
|
-
mimeType:
|
|
625
|
+
mimeType: "application/octet-stream",
|
|
627
626
|
};
|
|
628
627
|
await fileTable.saveFile(metadata, largeData);
|
|
629
628
|
const readStream = await fileTable.openReadStream(fileId);
|
|
630
629
|
expect(readStream).toBeTruthy();
|
|
631
630
|
// Read first chunk
|
|
632
|
-
const firstChunk = await readStream
|
|
631
|
+
const firstChunk = await readStream?.read(chunkSize);
|
|
633
632
|
expect(firstChunk?.length).toBe(chunkSize);
|
|
634
|
-
expect(firstChunk?.every(b => b === 65)).toBe(true); // All 'A's
|
|
633
|
+
expect(firstChunk?.every((b) => b === 65)).toBe(true); // All 'A's
|
|
635
634
|
// Read remainder
|
|
636
|
-
const remainder = await readStream
|
|
635
|
+
const remainder = await readStream?.read();
|
|
637
636
|
expect(remainder?.length).toBe(1000);
|
|
638
|
-
expect(remainder?.every(b => b === 66)).toBe(true); // All 'B's
|
|
637
|
+
expect(remainder?.every((b) => b === 66)).toBe(true); // All 'B's
|
|
639
638
|
// Should be at EOF
|
|
640
|
-
const eof = await readStream
|
|
639
|
+
const eof = await readStream?.read();
|
|
641
640
|
expect(eof).toBeNull();
|
|
642
|
-
expect(readStream
|
|
641
|
+
expect(readStream?.isEOF()).toBe(true);
|
|
643
642
|
});
|
|
644
|
-
it(
|
|
645
|
-
const readStream = await fileTable.openReadStream(
|
|
643
|
+
it("should return null for non-existent file", async () => {
|
|
644
|
+
const readStream = await fileTable.openReadStream("non-existent-file");
|
|
646
645
|
expect(readStream).toBeNull();
|
|
647
646
|
});
|
|
648
|
-
it(
|
|
647
|
+
it("should handle partial reads correctly", async () => {
|
|
649
648
|
const fileId = (0, utils_1.newid)();
|
|
650
|
-
const data = new Uint8Array(Buffer.from(
|
|
649
|
+
const data = new Uint8Array(Buffer.from("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "utf8"));
|
|
651
650
|
const metadata = {
|
|
652
651
|
fileId,
|
|
653
|
-
name:
|
|
652
|
+
name: "partial-read.txt",
|
|
654
653
|
fileSize: data.length,
|
|
655
|
-
mimeType:
|
|
654
|
+
mimeType: "text/plain",
|
|
656
655
|
};
|
|
657
656
|
await fileTable.saveFile(metadata, data);
|
|
658
657
|
const readStream = await fileTable.openReadStream(fileId);
|
|
659
658
|
expect(readStream).toBeTruthy();
|
|
660
659
|
// Read in small chunks
|
|
661
|
-
const chunk1 = await readStream
|
|
662
|
-
expect(Buffer.from(chunk1).toString(
|
|
663
|
-
expect(readStream
|
|
664
|
-
expect(readStream
|
|
665
|
-
const chunk2 = await readStream
|
|
666
|
-
expect(Buffer.from(chunk2).toString(
|
|
667
|
-
expect(readStream
|
|
668
|
-
const chunk3 = await readStream
|
|
669
|
-
expect(Buffer.from(chunk3).toString(
|
|
670
|
-
expect(readStream
|
|
671
|
-
expect(readStream
|
|
660
|
+
const chunk1 = await readStream?.read(5);
|
|
661
|
+
expect(Buffer.from(chunk1).toString("utf8")).toBe("ABCDE");
|
|
662
|
+
expect(readStream?.getPosition()).toBe(5);
|
|
663
|
+
expect(readStream?.getBytesRemaining()).toBe(21);
|
|
664
|
+
const chunk2 = await readStream?.read(10);
|
|
665
|
+
expect(Buffer.from(chunk2).toString("utf8")).toBe("FGHIJKLMNO");
|
|
666
|
+
expect(readStream?.getPosition()).toBe(15);
|
|
667
|
+
const chunk3 = await readStream?.read(20); // More than remaining
|
|
668
|
+
expect(Buffer.from(chunk3).toString("utf8")).toBe("PQRSTUVWXYZ");
|
|
669
|
+
expect(readStream?.getPosition()).toBe(26);
|
|
670
|
+
expect(readStream?.getBytesRemaining()).toBe(0);
|
|
672
671
|
// Should be EOF now
|
|
673
|
-
const eof = await readStream
|
|
672
|
+
const eof = await readStream?.read();
|
|
674
673
|
expect(eof).toBeNull();
|
|
675
674
|
});
|
|
676
|
-
it(
|
|
675
|
+
it("should return null when chunk is unavailable", async () => {
|
|
677
676
|
const fileId = (0, utils_1.newid)();
|
|
678
677
|
const metadata = {
|
|
679
678
|
fileId,
|
|
680
|
-
name:
|
|
679
|
+
name: "missing-chunk.txt",
|
|
681
680
|
fileSize: 50,
|
|
682
|
-
fileHash:
|
|
683
|
-
chunkHashes: [
|
|
681
|
+
fileHash: "hash123",
|
|
682
|
+
chunkHashes: ["missing_chunk_hash"],
|
|
684
683
|
};
|
|
685
684
|
// Insert file metadata without storing the actual chunk
|
|
686
685
|
await fileTable.dataSource.insert(metadata);
|
|
@@ -690,16 +689,16 @@ describe('FileTable', () => {
|
|
|
690
689
|
expect(result).toBeNull();
|
|
691
690
|
});
|
|
692
691
|
});
|
|
693
|
-
describe(
|
|
694
|
-
it(
|
|
692
|
+
describe("Round-trip streaming", () => {
|
|
693
|
+
it("should write and read back identical data using streams", async () => {
|
|
695
694
|
const fileId = (0, utils_1.newid)();
|
|
696
|
-
const testData =
|
|
697
|
-
const originalBuffer = new Uint8Array(Buffer.from(testData,
|
|
695
|
+
const testData = "This is a round-trip test with streaming! ".repeat(1000);
|
|
696
|
+
const originalBuffer = new Uint8Array(Buffer.from(testData, "utf8"));
|
|
698
697
|
const metadata = {
|
|
699
698
|
fileId,
|
|
700
|
-
name:
|
|
699
|
+
name: "roundtrip.txt",
|
|
701
700
|
fileSize: 0,
|
|
702
|
-
mimeType:
|
|
701
|
+
mimeType: "text/plain",
|
|
703
702
|
};
|
|
704
703
|
// Write using stream
|
|
705
704
|
const writeStream = await fileTable.createWriteStream(metadata);
|
|
@@ -717,11 +716,11 @@ describe('FileTable', () => {
|
|
|
717
716
|
// Read back using stream
|
|
718
717
|
const readStream = await fileTable.openReadStream(fileId);
|
|
719
718
|
expect(readStream).toBeTruthy();
|
|
720
|
-
const readBuffer = await readStream
|
|
719
|
+
const readBuffer = await readStream?.readAll();
|
|
721
720
|
// Verify identical
|
|
722
721
|
expect(readBuffer).not.toBeNull();
|
|
723
722
|
expect(new Uint8Array(readBuffer)).toEqual(originalBuffer);
|
|
724
|
-
expect(Buffer.from(readBuffer).toString(
|
|
723
|
+
expect(Buffer.from(readBuffer).toString("utf8")).toBe(testData);
|
|
725
724
|
});
|
|
726
725
|
});
|
|
727
726
|
});
|