@tstdl/base 0.93.127 → 0.93.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/client/client.js +45 -9
- package/api/client/tests/api-client.test.d.ts +1 -0
- package/api/client/tests/api-client.test.js +194 -0
- package/api/types.d.ts +35 -3
- package/authentication/client/authentication.service.js +30 -11
- package/authentication/client/http-client.middleware.js +10 -3
- package/authentication/server/authentication.service.d.ts +12 -0
- package/authentication/server/authentication.service.js +14 -2
- package/authentication/tests/authentication.client-error-handling.test.js +23 -66
- package/authentication/tests/authentication.client-service-refresh.test.js +14 -14
- package/cancellation/token.d.ts +6 -0
- package/cancellation/token.js +8 -0
- package/document-management/server/services/document-file.service.js +10 -9
- package/document-management/server/services/document-management-ancillary.service.d.ts +12 -1
- package/document-management/server/services/document-management-ancillary.service.js +9 -0
- package/file/server/temporary-file.d.ts +2 -1
- package/file/server/temporary-file.js +5 -1
- package/http/client/adapters/undici.adapter.js +0 -2
- package/http/client/http-client-request.d.ts +2 -0
- package/http/client/http-client-request.js +4 -0
- package/http/client/http-client-response.d.ts +1 -1
- package/http/client/http-client-response.js +3 -2
- package/http/utils.d.ts +6 -0
- package/http/utils.js +71 -0
- package/injector/injector.js +2 -0
- package/mail/drizzle/0000_numerous_the_watchers.sql +8 -0
- package/mail/drizzle/meta/0000_snapshot.json +1 -32
- package/mail/drizzle/meta/_journal.json +2 -9
- package/object-storage/s3/s3.object-storage.js +6 -6
- package/object-storage/s3/tests/s3.object-storage.integration.test.js +22 -53
- package/orm/tests/repository-expiration.test.js +3 -3
- package/package.json +1 -1
- package/pdf/utils.d.ts +24 -3
- package/pdf/utils.js +89 -30
- package/process/spawn.d.ts +1 -1
- package/rate-limit/tests/postgres-rate-limiter.test.js +9 -7
- package/renderer/typst.d.ts +5 -0
- package/renderer/typst.js +9 -5
- package/task-queue/tests/complex.test.js +22 -22
- package/task-queue/tests/dependencies.test.js +15 -13
- package/task-queue/tests/queue.test.js +13 -13
- package/task-queue/tests/worker.test.js +12 -12
- package/testing/integration-setup.d.ts +2 -0
- package/testing/integration-setup.js +13 -7
- package/utils/backoff.d.ts +27 -3
- package/utils/backoff.js +31 -9
- package/utils/index.d.ts +1 -0
- package/utils/index.js +1 -0
- package/utils/retry-with-backoff.d.ts +22 -0
- package/utils/retry-with-backoff.js +64 -0
- package/utils/tests/backoff.test.d.ts +1 -0
- package/utils/tests/backoff.test.js +41 -0
- package/utils/tests/retry-with-backoff.test.d.ts +1 -0
- package/utils/tests/retry-with-backoff.test.js +49 -0
- package/mail/drizzle/0000_previous_malcolm_colcord.sql +0 -13
- package/mail/drizzle/0001_flimsy_bloodscream.sql +0 -5
- package/mail/drizzle/meta/0001_snapshot.json +0 -69
|
@@ -240,16 +240,16 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
|
|
|
240
240
|
async getDownloadUrl(key, expirationTimestamp, responseHeaders) {
|
|
241
241
|
const bucketKey = this.getBucketKey(key);
|
|
242
242
|
const expiration = getExpiration(expirationTimestamp);
|
|
243
|
-
const expiresHeader = responseHeaders?.['Expires'];
|
|
243
|
+
const expiresHeader = responseHeaders?.['Expires'] ?? responseHeaders?.['Response-Expires'];
|
|
244
244
|
const mappedExpiresHeader = isDefined(expiresHeader) ? (isDate(expiresHeader) ? expiresHeader : new Date(expiresHeader)) : undefined;
|
|
245
245
|
return await getSignedUrl(this.client, new GetObjectCommand({
|
|
246
246
|
Bucket: this.bucket,
|
|
247
247
|
Key: bucketKey,
|
|
248
|
-
ResponseContentType: responseHeaders?.['Content-Type'],
|
|
249
|
-
ResponseContentDisposition: responseHeaders?.['Content-Disposition'],
|
|
250
|
-
ResponseCacheControl: responseHeaders?.['Cache-Control'],
|
|
251
|
-
ResponseContentLanguage: responseHeaders?.['Content-Language'],
|
|
252
|
-
ResponseContentEncoding: responseHeaders?.['Content-Encoding'],
|
|
248
|
+
ResponseContentType: responseHeaders?.['Content-Type'] ?? responseHeaders?.['Response-Content-Type'],
|
|
249
|
+
ResponseContentDisposition: responseHeaders?.['Content-Disposition'] ?? responseHeaders?.['Response-Content-Disposition'],
|
|
250
|
+
ResponseCacheControl: responseHeaders?.['Cache-Control'] ?? responseHeaders?.['Response-Cache-Control'],
|
|
251
|
+
ResponseContentLanguage: responseHeaders?.['Content-Language'] ?? responseHeaders?.['Response-Content-Language'],
|
|
252
|
+
ResponseContentEncoding: responseHeaders?.['Content-Encoding'] ?? responseHeaders?.['Response-Content-Encoding'],
|
|
253
253
|
ResponseExpires: mappedExpiresHeader,
|
|
254
254
|
}), { expiresIn: expiration });
|
|
255
255
|
}
|
|
@@ -1,27 +1,20 @@
|
|
|
1
1
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
2
2
|
import { setupIntegrationTest } from '../../../testing/index.js';
|
|
3
3
|
import { readBinaryStream } from '../../../utils/stream/stream-reader.js';
|
|
4
|
-
import {
|
|
4
|
+
import { isUndefined } from '../../../utils/type-guards.js';
|
|
5
5
|
import { S3ObjectStorage } from '../s3.object-storage.js';
|
|
6
6
|
describe('S3ObjectStorage Integration', () => {
|
|
7
7
|
let storage;
|
|
8
|
-
const bucketName = 'integration-test-bucket';
|
|
9
8
|
beforeAll(async () => {
|
|
10
9
|
const { injector } = await setupIntegrationTest({
|
|
11
10
|
modules: { objectStorage: true },
|
|
12
11
|
});
|
|
13
|
-
configureS3ObjectStorage({
|
|
14
|
-
endpoint: 'http://127.0.0.1:9000',
|
|
15
|
-
accessKey: 'tstdl-dev',
|
|
16
|
-
secretKey: 'tstdl-dev',
|
|
17
|
-
bucket: bucketName,
|
|
18
|
-
region: 'us-east-1',
|
|
19
|
-
forcePathStyle: true,
|
|
20
|
-
injector,
|
|
21
|
-
});
|
|
22
12
|
storage = await injector.resolveAsync(S3ObjectStorage, 'test-module');
|
|
23
13
|
});
|
|
24
14
|
afterAll(async () => {
|
|
15
|
+
if (isUndefined(storage)) {
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
25
18
|
const objects = await storage.getObjects();
|
|
26
19
|
for (const obj of objects) {
|
|
27
20
|
await storage.deleteObject(obj.key);
|
|
@@ -80,7 +73,7 @@ describe('S3ObjectStorage Integration', () => {
|
|
|
80
73
|
const key = 'signed-download.txt';
|
|
81
74
|
await storage.uploadObject(key, new TextEncoder().encode('signed download'));
|
|
82
75
|
const url = await storage.getDownloadUrl(key, Date.now() + 60000);
|
|
83
|
-
expect(url).
|
|
76
|
+
expect(url).toMatch(/http:\/\/(127\.0\.0\.1|localhost):9000/);
|
|
84
77
|
const response = await fetch(url);
|
|
85
78
|
expect(response.status).toBe(200);
|
|
86
79
|
expect(await response.text()).toBe('signed download');
|
|
@@ -154,15 +147,8 @@ describe('S3ObjectStorage Integration', () => {
|
|
|
154
147
|
expect(stat.metadata['extra-key']).toBe('extra-value');
|
|
155
148
|
});
|
|
156
149
|
it('should work with bucket per module', async () => {
|
|
157
|
-
const { injector } = await setupIntegrationTest(
|
|
158
|
-
|
|
159
|
-
endpoint: 'http://127.0.0.1:9000',
|
|
160
|
-
accessKey: 'tstdl-dev',
|
|
161
|
-
secretKey: 'tstdl-dev',
|
|
162
|
-
bucketPerModule: true,
|
|
163
|
-
region: 'us-east-1',
|
|
164
|
-
forcePathStyle: true,
|
|
165
|
-
injector,
|
|
150
|
+
const { injector } = await setupIntegrationTest({
|
|
151
|
+
modules: { objectStorage: true },
|
|
166
152
|
});
|
|
167
153
|
const moduleName = `test-bucket-per-module-${Math.floor(Math.random() * 1000000)}`;
|
|
168
154
|
const perModuleStorage = await injector.resolveAsync(S3ObjectStorage, moduleName);
|
|
@@ -177,7 +163,7 @@ describe('S3ObjectStorage Integration', () => {
|
|
|
177
163
|
const metadata = { 's3-test': 'true' };
|
|
178
164
|
await storage.uploadObject(key, content, { metadata });
|
|
179
165
|
const obj = await storage.getObject(key);
|
|
180
|
-
expect(await obj.getResourceUri()).toBe(`s3://
|
|
166
|
+
expect(await obj.getResourceUri()).toBe(`s3://test-module/${key}`);
|
|
181
167
|
expect(await obj.getContentLength()).toBe(content.length);
|
|
182
168
|
expect(await obj.getMetadata()).toMatchObject(metadata);
|
|
183
169
|
expect(new TextDecoder().decode(await obj.getContent())).toBe('s3 object');
|
|
@@ -242,25 +228,22 @@ describe('S3ObjectStorage Integration', () => {
|
|
|
242
228
|
const key = 'signed-download-expires.txt';
|
|
243
229
|
await storage.uploadObject(key, new TextEncoder().encode('signed download expires'));
|
|
244
230
|
const url = await storage.getDownloadUrl(key, Date.now() + 60000, {
|
|
245
|
-
|
|
231
|
+
Expires: new Date(Date.now() + 60000).toUTCString(),
|
|
246
232
|
});
|
|
247
|
-
expect(url).
|
|
233
|
+
expect(url).toMatch(/http:\/\/(127\.0\.0\.1|localhost):9000/);
|
|
248
234
|
const response = await fetch(url);
|
|
249
235
|
expect(response.status).toBe(200);
|
|
250
236
|
});
|
|
251
237
|
it('should handle Forbidden error in ensureBucketExists with wrong credentials', async () => {
|
|
252
|
-
const { injector } = await setupIntegrationTest(
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
region: 'us-east-1',
|
|
259
|
-
forcePathStyle: true,
|
|
260
|
-
injector,
|
|
238
|
+
const { injector } = await setupIntegrationTest({
|
|
239
|
+
modules: { objectStorage: true },
|
|
240
|
+
s3: {
|
|
241
|
+
accessKey: 'wrong',
|
|
242
|
+
secretKey: 'wrong',
|
|
243
|
+
},
|
|
261
244
|
});
|
|
262
245
|
try {
|
|
263
|
-
await injector.resolveAsync(S3ObjectStorage, '
|
|
246
|
+
await injector.resolveAsync(S3ObjectStorage, 'forbidden-bucket');
|
|
264
247
|
expect.fail('Should have thrown');
|
|
265
248
|
}
|
|
266
249
|
catch (error) {
|
|
@@ -268,17 +251,10 @@ describe('S3ObjectStorage Integration', () => {
|
|
|
268
251
|
}
|
|
269
252
|
});
|
|
270
253
|
it('should copy object between different storages', async () => {
|
|
271
|
-
const { injector } = await setupIntegrationTest(
|
|
272
|
-
|
|
273
|
-
endpoint: 'http://127.0.0.1:9000',
|
|
274
|
-
accessKey: 'tstdl-dev',
|
|
275
|
-
secretKey: 'tstdl-dev',
|
|
276
|
-
bucket: 'another-bucket',
|
|
277
|
-
region: 'us-east-1',
|
|
278
|
-
forcePathStyle: true,
|
|
279
|
-
injector,
|
|
254
|
+
const { injector } = await setupIntegrationTest({
|
|
255
|
+
modules: { objectStorage: true },
|
|
280
256
|
});
|
|
281
|
-
const anotherStorage = await injector.resolveAsync(S3ObjectStorage, 'another-
|
|
257
|
+
const anotherStorage = await injector.resolveAsync(S3ObjectStorage, 'another-bucket');
|
|
282
258
|
const sourceKey = 'cross-storage-source.txt';
|
|
283
259
|
const destKey = 'cross-storage-dest.txt';
|
|
284
260
|
await storage.uploadObject(sourceKey, new TextEncoder().encode('cross storage content'));
|
|
@@ -288,15 +264,8 @@ describe('S3ObjectStorage Integration', () => {
|
|
|
288
264
|
expect(new TextDecoder().decode(content)).toBe('cross storage content');
|
|
289
265
|
});
|
|
290
266
|
it('should cover ensureBucketExists with region', async () => {
|
|
291
|
-
const { injector } = await setupIntegrationTest(
|
|
292
|
-
|
|
293
|
-
endpoint: 'http://127.0.0.1:9000',
|
|
294
|
-
accessKey: 'tstdl-dev',
|
|
295
|
-
secretKey: 'tstdl-dev',
|
|
296
|
-
bucketPerModule: true,
|
|
297
|
-
region: 'us-east-1',
|
|
298
|
-
forcePathStyle: true,
|
|
299
|
-
injector,
|
|
267
|
+
const { injector } = await setupIntegrationTest({
|
|
268
|
+
modules: { objectStorage: true },
|
|
300
269
|
});
|
|
301
270
|
const perModuleStorage = await injector.resolveAsync(S3ObjectStorage, `region-test-${Math.floor(Math.random() * 1000000)}`);
|
|
302
271
|
await perModuleStorage.ensureBucketExists('us-east-1', { objectLocking: true });
|
|
@@ -51,7 +51,7 @@ describe('ORM Repository Expiration', () => {
|
|
|
51
51
|
], TtlEntity.prototype, "name", void 0);
|
|
52
52
|
TtlEntity = __decorate([
|
|
53
53
|
Table('ttl_entities', { schema }),
|
|
54
|
-
TimeToLive(
|
|
54
|
+
TimeToLive(100, 'hard') // 100ms TTL
|
|
55
55
|
], TtlEntity);
|
|
56
56
|
beforeAll(async () => {
|
|
57
57
|
injector = new Injector('Test');
|
|
@@ -95,8 +95,8 @@ describe('ORM Repository Expiration', () => {
|
|
|
95
95
|
await runInInjectionContext(injector, async () => {
|
|
96
96
|
const repository = injectRepository(TtlEntity);
|
|
97
97
|
const e1 = await repository.insert(Object.assign(new TtlEntity(), { name: 'Valid' }));
|
|
98
|
-
// Wait
|
|
99
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
98
|
+
// Wait 150ms for expiration
|
|
99
|
+
await new Promise((resolve) => setTimeout(resolve, 150));
|
|
100
100
|
await repository.processExpirations();
|
|
101
101
|
const all = await repository.loadAll({ withDeleted: true });
|
|
102
102
|
expect(all).toHaveLength(0);
|
package/package.json
CHANGED
package/pdf/utils.d.ts
CHANGED
|
@@ -1,6 +1,23 @@
|
|
|
1
|
-
|
|
2
|
-
export
|
|
3
|
-
|
|
1
|
+
import { TemporaryFile } from '../file/server/temporary-file.js';
|
|
2
|
+
export type PdfInput = string | {
|
|
3
|
+
file: string;
|
|
4
|
+
} | Uint8Array | ReadableStream<Uint8Array>;
|
|
5
|
+
export type PdfInputWithPageRange = string | {
|
|
6
|
+
file: string;
|
|
7
|
+
} | Uint8Array | ReadableStream<Uint8Array> | {
|
|
8
|
+
input: PdfInput;
|
|
9
|
+
pages: PageRange[];
|
|
10
|
+
};
|
|
11
|
+
export type PageRange = number | {
|
|
12
|
+
from?: number;
|
|
13
|
+
to?: number;
|
|
14
|
+
};
|
|
15
|
+
export declare function getPdfPageCount(file: PdfInput): Promise<number>;
|
|
16
|
+
export declare function extractPdfPages(file: PdfInput, pages: PageRange[]): Promise<TemporaryFile>;
|
|
17
|
+
export declare function mergePdfs(sourceFiles: PdfInputWithPageRange[]): Promise<TemporaryFile>;
|
|
18
|
+
export declare function overlayPdfs(base: PdfInput, overlay: PdfInput, options?: {
|
|
19
|
+
repeat?: boolean;
|
|
20
|
+
}): Promise<TemporaryFile>;
|
|
4
21
|
/**
|
|
5
22
|
* Convert a PDF page to an image.
|
|
6
23
|
* @param file The PDF file to convert.
|
|
@@ -10,3 +27,7 @@ export declare function mergePdfsStream(pdfs: (string | Uint8Array | ReadableStr
|
|
|
10
27
|
* @returns The converted image as a byte array.
|
|
11
28
|
*/
|
|
12
29
|
export declare function pdfToImage(file: string | Uint8Array | ReadableStream<Uint8Array>, page: number, size: number, format: 'png' | 'jpeg' | 'tiff' | 'ps' | 'eps' | 'pdf' | 'svg'): Promise<Uint8Array>;
|
|
30
|
+
export declare function isInputWithPageRange(input: PdfInputWithPageRange): input is {
|
|
31
|
+
input: PdfInput;
|
|
32
|
+
pages: PageRange[];
|
|
33
|
+
};
|
package/pdf/utils.js
CHANGED
|
@@ -51,15 +51,15 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
53
|
import { TemporaryFile } from '../file/server/temporary-file.js';
|
|
54
|
-
import { spawnCommand } from '../process/spawn.js';
|
|
55
|
-
import {
|
|
54
|
+
import { spawnCommand, spawnWaitReadCommand } from '../process/spawn.js';
|
|
55
|
+
import { hasOwnProperty } from '../utils/object/object.js';
|
|
56
|
+
import { isNotString, isNumber, isObject, isReadableStream, isString, isUint8Array, isUndefined } from '../utils/type-guards.js';
|
|
56
57
|
export async function getPdfPageCount(file) {
|
|
57
58
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
58
59
|
try {
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
const
|
|
62
|
-
const process = await spawnCommand('qpdf', ['--show-npages', path]);
|
|
60
|
+
const stack = __addDisposableResource(env_1, new AsyncDisposableStack(), true);
|
|
61
|
+
const [sourceFile] = await getPdfSourceFiles([file], stack);
|
|
62
|
+
const process = await spawnCommand('qpdf', ['--show-npages', sourceFile]);
|
|
63
63
|
const { code } = await process.wait();
|
|
64
64
|
if (code != 0) {
|
|
65
65
|
const errorOutput = await process.readError();
|
|
@@ -78,14 +78,19 @@ export async function getPdfPageCount(file) {
|
|
|
78
78
|
await result_1;
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
|
-
export async function
|
|
81
|
+
export async function extractPdfPages(file, pages) {
|
|
82
82
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
83
83
|
try {
|
|
84
84
|
const stack = __addDisposableResource(env_2, new AsyncDisposableStack(), true);
|
|
85
|
-
const
|
|
86
|
-
const
|
|
87
|
-
|
|
88
|
-
|
|
85
|
+
const [sourceFile] = await getPdfSourceFiles([file], stack);
|
|
86
|
+
const resultFile = TemporaryFile.create();
|
|
87
|
+
const pagesString = toQpdfPageRangeString(pages);
|
|
88
|
+
const processResult = await spawnWaitReadCommand('string', 'qpdf', ['--empty', '--pages', sourceFile, pagesString, '--', resultFile.path]);
|
|
89
|
+
if (processResult.code != 0) {
|
|
90
|
+
await resultFile.dispose();
|
|
91
|
+
throw new Error(processResult.error.trim());
|
|
92
|
+
}
|
|
93
|
+
return resultFile;
|
|
89
94
|
}
|
|
90
95
|
catch (e_2) {
|
|
91
96
|
env_2.error = e_2;
|
|
@@ -97,14 +102,26 @@ export async function mergePdfs(pdfs) {
|
|
|
97
102
|
await result_2;
|
|
98
103
|
}
|
|
99
104
|
}
|
|
100
|
-
export async function
|
|
105
|
+
export async function mergePdfs(sourceFiles) {
|
|
101
106
|
const env_3 = { stack: [], error: void 0, hasError: false };
|
|
102
107
|
try {
|
|
103
108
|
const stack = __addDisposableResource(env_3, new AsyncDisposableStack(), true);
|
|
104
109
|
const resultFile = __addDisposableResource(env_3, TemporaryFile.create(), true);
|
|
105
|
-
const
|
|
106
|
-
|
|
107
|
-
|
|
110
|
+
const sourceFilePaths = await getPdfSourceFiles(sourceFiles, stack);
|
|
111
|
+
const pages = sourceFiles.map((source) => isInputWithPageRange(source) ? source.pages : undefined);
|
|
112
|
+
const sourceArguments = sourceFilePaths.flatMap((path, index) => {
|
|
113
|
+
const pageRanges = pages[index];
|
|
114
|
+
if (isUndefined(pageRanges)) {
|
|
115
|
+
return [path];
|
|
116
|
+
}
|
|
117
|
+
const pagesString = toQpdfPageRangeString(pageRanges);
|
|
118
|
+
return [path, pagesString];
|
|
119
|
+
});
|
|
120
|
+
const processResult = await spawnWaitReadCommand('string', 'qpdf', ['--empty', '--pages', ...sourceArguments, '--', resultFile.path]);
|
|
121
|
+
if (processResult.code != 0) {
|
|
122
|
+
throw new Error(processResult.error);
|
|
123
|
+
}
|
|
124
|
+
return resultFile;
|
|
108
125
|
}
|
|
109
126
|
catch (e_3) {
|
|
110
127
|
env_3.error = e_3;
|
|
@@ -116,22 +133,32 @@ export async function mergePdfsStream(pdfs) {
|
|
|
116
133
|
await result_3;
|
|
117
134
|
}
|
|
118
135
|
}
|
|
119
|
-
async function
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
136
|
+
export async function overlayPdfs(base, overlay, options) {
|
|
137
|
+
const env_4 = { stack: [], error: void 0, hasError: false };
|
|
138
|
+
try {
|
|
139
|
+
const stack = __addDisposableResource(env_4, new AsyncDisposableStack(), true);
|
|
140
|
+
const [baseFilePath, overlayFilePath] = await getPdfSourceFiles([base, overlay], stack);
|
|
141
|
+
const resultFile = TemporaryFile.create();
|
|
142
|
+
stack.use(resultFile);
|
|
143
|
+
const args = [baseFilePath, '--overlay', overlayFilePath];
|
|
144
|
+
if (options?.repeat) {
|
|
145
|
+
args.push('--repeat=1');
|
|
123
146
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
147
|
+
args.push('--', resultFile.path);
|
|
148
|
+
const processResult = await spawnWaitReadCommand('string', 'qpdf', args);
|
|
149
|
+
if (processResult.code != 0) {
|
|
150
|
+
throw new Error(processResult.error);
|
|
151
|
+
}
|
|
152
|
+
return resultFile;
|
|
153
|
+
}
|
|
154
|
+
catch (e_4) {
|
|
155
|
+
env_4.error = e_4;
|
|
156
|
+
env_4.hasError = true;
|
|
157
|
+
}
|
|
158
|
+
finally {
|
|
159
|
+
const result_4 = __disposeResources(env_4);
|
|
160
|
+
if (result_4)
|
|
161
|
+
await result_4;
|
|
135
162
|
}
|
|
136
163
|
}
|
|
137
164
|
/**
|
|
@@ -151,3 +178,35 @@ export async function pdfToImage(file, page, size, format) {
|
|
|
151
178
|
}
|
|
152
179
|
return await process.readOutputBytes();
|
|
153
180
|
}
|
|
181
|
+
function toQpdfPageRangeString(pages) {
|
|
182
|
+
return pages.map((page) => {
|
|
183
|
+
if (isNumber(page)) {
|
|
184
|
+
return String(page);
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
const from = page.from ?? '1';
|
|
188
|
+
const to = page.to ?? 'z';
|
|
189
|
+
return `${from}-${to}`;
|
|
190
|
+
}
|
|
191
|
+
}).join(',');
|
|
192
|
+
}
|
|
193
|
+
async function getPdfSourceFiles(input, stack) {
|
|
194
|
+
return await Promise.all(input.map(async (pdf) => {
|
|
195
|
+
if (isUint8Array(pdf) || isReadableStream(pdf)) {
|
|
196
|
+
const tmpFile = await TemporaryFile.from(pdf);
|
|
197
|
+
stack.use(tmpFile);
|
|
198
|
+
return tmpFile.path;
|
|
199
|
+
}
|
|
200
|
+
if (isString(pdf)) {
|
|
201
|
+
return pdf;
|
|
202
|
+
}
|
|
203
|
+
if (isInputWithPageRange(pdf)) {
|
|
204
|
+
const [path] = await getPdfSourceFiles([pdf.input], stack);
|
|
205
|
+
return path;
|
|
206
|
+
}
|
|
207
|
+
return pdf.file;
|
|
208
|
+
}));
|
|
209
|
+
}
|
|
210
|
+
export function isInputWithPageRange(input) {
|
|
211
|
+
return isObject(input) && hasOwnProperty(input, 'input') && hasOwnProperty(input, 'pages');
|
|
212
|
+
}
|
package/process/spawn.d.ts
CHANGED
|
@@ -18,7 +18,7 @@ export type SpawnOptions = {
|
|
|
18
18
|
workingDirectory?: string;
|
|
19
19
|
environment?: Record<string, string>;
|
|
20
20
|
};
|
|
21
|
-
export type SpawnCommandResult = TransformStream<Uint8Array, Uint8Array
|
|
21
|
+
export type SpawnCommandResult = TransformStream<Uint8Array, Uint8Array<ArrayBuffer>> & {
|
|
22
22
|
process: ChildProcessWithoutNullStreams;
|
|
23
23
|
stderr: ReadableStream<Uint8Array>;
|
|
24
24
|
write(chunk: ReadableStream<Uint8Array> | Uint8Array | string, options?: StreamPipeOptions): Promise<void>;
|
|
@@ -11,7 +11,7 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
11
11
|
const rateLimiterProvider = injector.resolve(RateLimiterProvider);
|
|
12
12
|
rateLimiter = rateLimiterProvider.get(limiterName, {
|
|
13
13
|
burstCapacity: 10,
|
|
14
|
-
refillInterval:
|
|
14
|
+
refillInterval: 500, // 10 tokens per 500ms -> 1 token per 50ms
|
|
15
15
|
});
|
|
16
16
|
});
|
|
17
17
|
afterAll(async () => {
|
|
@@ -30,9 +30,9 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
30
30
|
const resource = 'res-2';
|
|
31
31
|
await rateLimiter.tryAcquire(resource, 10);
|
|
32
32
|
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(false);
|
|
33
|
-
// Wait for
|
|
34
|
-
await timeout(
|
|
35
|
-
expect(await rateLimiter.tryAcquire(resource,
|
|
33
|
+
// Wait for 1 token (50ms) + buffer
|
|
34
|
+
await timeout(75);
|
|
35
|
+
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(true);
|
|
36
36
|
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(false);
|
|
37
37
|
});
|
|
38
38
|
it('should refund tokens', async () => {
|
|
@@ -55,7 +55,9 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
55
55
|
await rateLimiter.tryAcquire(resource, 0);
|
|
56
56
|
const results = await Promise.all(Array.from({ length: 20 }).map(() => rateLimiter.tryAcquire(resource, 1)));
|
|
57
57
|
const successCount = results.filter(Boolean).length;
|
|
58
|
-
expect
|
|
58
|
+
// We expect 10, but allow up to 12 if tokens refilled during the Promise.all
|
|
59
|
+
expect(successCount).toBeGreaterThanOrEqual(10);
|
|
60
|
+
expect(successCount).toBeLessThanOrEqual(12);
|
|
59
61
|
}, 15000);
|
|
60
62
|
it('should always allow zero or negative cost', async () => {
|
|
61
63
|
const resource = 'res-zero';
|
|
@@ -72,8 +74,8 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
72
74
|
// Drain
|
|
73
75
|
await rateLimiter.tryAcquire(resource, 10);
|
|
74
76
|
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(false);
|
|
75
|
-
// Wait for full refill (
|
|
76
|
-
await timeout(
|
|
77
|
+
// Wait for full refill (500ms) + extra
|
|
78
|
+
await timeout(600);
|
|
77
79
|
// Should only have 10 tokens
|
|
78
80
|
expect(await rateLimiter.tryAcquire(resource, 10)).toBe(true);
|
|
79
81
|
// Should be empty again immediately
|
package/renderer/typst.d.ts
CHANGED
|
@@ -43,6 +43,11 @@ export type TypstRenderOptions = {
|
|
|
43
43
|
* One (or multiple comma-separated) PDF standards that Typst will enforce conformance with.
|
|
44
44
|
*/
|
|
45
45
|
pdfStandard?: LiteralUnion<'1.4' | '1.5' | '1.6' | '1.7' | '2.0' | 'a-1b' | 'a-1a' | 'a-2b' | 'a-2u' | 'a-2a' | 'a-3b' | 'a-3u' | 'a-3a' | 'a-4' | 'a-4f' | 'a-4e' | 'ua-1', string>;
|
|
46
|
+
/**
|
|
47
|
+
* A reference file to use when rendering to docx format. This allows customizing styles, fonts, etc. in the generated docx file. The file should be a valid docx file that serves as a template for the output.
|
|
48
|
+
* The reference file can be generated with `pandoc --print-default-data-file reference.docx`
|
|
49
|
+
*/
|
|
50
|
+
docxReferenceFile?: string;
|
|
46
51
|
};
|
|
47
52
|
/**
|
|
48
53
|
* Renders Typst source code to a file in the specified format.
|
package/renderer/typst.js
CHANGED
|
@@ -14,10 +14,9 @@ import { isDefined, isNumber, isString } from '../utils/type-guards.js';
|
|
|
14
14
|
export async function renderTypst(source, options) {
|
|
15
15
|
const format = options?.format ?? 'pdf';
|
|
16
16
|
const command = (format == 'docx') ? 'pandoc' : 'typst';
|
|
17
|
-
let args =
|
|
18
|
-
? ['--from', 'typst', '--to', 'docx', '--output']
|
|
19
|
-
: ['compile', '--format', format];
|
|
17
|
+
let args = [];
|
|
20
18
|
if (command == 'typst') {
|
|
19
|
+
args = ['compile', '--format', format];
|
|
21
20
|
if (isDefined(options?.root)) {
|
|
22
21
|
args.push('--root', options.root);
|
|
23
22
|
}
|
|
@@ -45,14 +44,19 @@ export async function renderTypst(source, options) {
|
|
|
45
44
|
args.push('--pages', pageParts.join(','));
|
|
46
45
|
}
|
|
47
46
|
}
|
|
48
|
-
|
|
47
|
+
else if (command == 'pandoc') {
|
|
48
|
+
args = ['--from', 'typst', '--to', 'docx', '--output', '-'];
|
|
49
|
+
if (isDefined(options?.docxReferenceFile)) {
|
|
50
|
+
args.push('--reference-doc', options.docxReferenceFile);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
49
53
|
const process = await spawnCommand(command, args);
|
|
50
54
|
const [{ code, output, error }] = await Promise.all([
|
|
51
55
|
process.waitRead('binary', { throwOnNonZeroExitCode: false }),
|
|
52
56
|
process.write(source),
|
|
53
57
|
]);
|
|
54
58
|
const errorString = decodeText(error);
|
|
55
|
-
if (code
|
|
59
|
+
if (code != 0) {
|
|
56
60
|
throw new Error(`
|
|
57
61
|
Typst compilation failed with exit code ${code}.\n
|
|
58
62
|
Error Output:\n${errorString}
|
|
@@ -15,13 +15,13 @@ describe('Complex Queue Scenarios', () => {
|
|
|
15
15
|
// Configure with specific settings for testing logic
|
|
16
16
|
queue = queueProvider.get(queueName, {
|
|
17
17
|
visibilityTimeout: 1000,
|
|
18
|
-
priorityAgingInterval:
|
|
18
|
+
priorityAgingInterval: 50, // Fast aging
|
|
19
19
|
priorityAgingStep: 10,
|
|
20
20
|
rateLimit: 5,
|
|
21
|
-
rateInterval:
|
|
22
|
-
retryDelayMinimum:
|
|
21
|
+
rateInterval: 50,
|
|
22
|
+
retryDelayMinimum: 50,
|
|
23
23
|
retryDelayGrowth: 2,
|
|
24
|
-
retention:
|
|
24
|
+
retention: 50, // Fast retention for archive test
|
|
25
25
|
});
|
|
26
26
|
});
|
|
27
27
|
afterEach(async () => {
|
|
@@ -33,12 +33,12 @@ describe('Complex Queue Scenarios', () => {
|
|
|
33
33
|
await injector?.dispose();
|
|
34
34
|
});
|
|
35
35
|
async function waitForStatus(id, status) {
|
|
36
|
-
for (let i = 0; i <
|
|
36
|
+
for (let i = 0; i < 50; i++) {
|
|
37
37
|
const task = await queue.getTask(id);
|
|
38
38
|
if (task?.status === status)
|
|
39
39
|
return;
|
|
40
40
|
await queue.processPendingFanIn();
|
|
41
|
-
await timeout(
|
|
41
|
+
await timeout(10);
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
44
|
describe('Complex Dependencies', () => {
|
|
@@ -62,7 +62,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
62
62
|
// Process B
|
|
63
63
|
const dB = await queue.dequeue({ types: ['B'] });
|
|
64
64
|
await queue.complete(dB);
|
|
65
|
-
await timeout(
|
|
65
|
+
await timeout(20);
|
|
66
66
|
await queue.processPendingFanIn();
|
|
67
67
|
// D still waiting (needs C)
|
|
68
68
|
const uD2 = await queue.getTask(taskD.id);
|
|
@@ -105,8 +105,8 @@ describe('Complex Queue Scenarios', () => {
|
|
|
105
105
|
describe('Scheduling & Priorities', () => {
|
|
106
106
|
it('should promote priority of old pending tasks (Aging)', async () => {
|
|
107
107
|
const t1 = await queue.enqueue('low', {}, { priority: 2000 });
|
|
108
|
-
// Wait for aging interval (
|
|
109
|
-
await timeout(
|
|
108
|
+
// Wait for aging interval (50ms)
|
|
109
|
+
await timeout(60);
|
|
110
110
|
await queue.maintenance();
|
|
111
111
|
const updated = await queue.getTask(t1.id);
|
|
112
112
|
// Default step is 10. 2000 - 10 = 1990
|
|
@@ -120,7 +120,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
120
120
|
const u1 = await queue.getTask(task.id);
|
|
121
121
|
expect(u1?.tries).toBe(1);
|
|
122
122
|
const delay1 = u1.scheduleTimestamp - currentTimestamp();
|
|
123
|
-
expect(delay1).toBeGreaterThan(
|
|
123
|
+
expect(delay1).toBeGreaterThan(20); // Approx check
|
|
124
124
|
// Force reschedule to now
|
|
125
125
|
await queue.reschedule(task.id, currentTimestamp());
|
|
126
126
|
// Try 2
|
|
@@ -129,12 +129,12 @@ describe('Complex Queue Scenarios', () => {
|
|
|
129
129
|
const u2 = await queue.getTask(task.id);
|
|
130
130
|
expect(u2?.tries).toBe(2);
|
|
131
131
|
const now = currentTimestamp();
|
|
132
|
-
expect(u2.scheduleTimestamp > now +
|
|
132
|
+
expect(u2.scheduleTimestamp > now + 50).toBe(true);
|
|
133
133
|
});
|
|
134
134
|
});
|
|
135
135
|
describe('Rate Limiting & Concurrency', () => {
|
|
136
136
|
it('should limit burst dequeue rate', async () => {
|
|
137
|
-
// Rate limit 5, interval
|
|
137
|
+
// Rate limit 5, interval 100ms
|
|
138
138
|
await queue.enqueueMany(Array.from({ length: 10 }, (_, i) => ({ type: 'burst', data: { i } })));
|
|
139
139
|
// Request burstCapacity (5)
|
|
140
140
|
const batch1 = await queue.dequeueMany(5);
|
|
@@ -143,7 +143,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
143
143
|
const batch2 = await queue.dequeueMany(1);
|
|
144
144
|
expect(batch2.length).toBe(0); // Rate limited
|
|
145
145
|
// Wait for refill
|
|
146
|
-
await timeout(
|
|
146
|
+
await timeout(60);
|
|
147
147
|
const batch3 = await queue.dequeueMany(5);
|
|
148
148
|
expect(batch3.length).toBe(5); // Refilled
|
|
149
149
|
});
|
|
@@ -179,8 +179,8 @@ describe('Complex Queue Scenarios', () => {
|
|
|
179
179
|
expect(before).toBeDefined();
|
|
180
180
|
expect(before?.status).toBe(TaskStatus.Completed);
|
|
181
181
|
expect(before.completeTimestamp > 0).toBe(true);
|
|
182
|
-
// Wait for retention (
|
|
183
|
-
await timeout(
|
|
182
|
+
// Wait for retention (50ms).
|
|
183
|
+
await timeout(60);
|
|
184
184
|
await archiveQueue.maintenance();
|
|
185
185
|
// Should move from main table to archive
|
|
186
186
|
const loaded = await archiveQueue.getTask(task.id);
|
|
@@ -189,9 +189,9 @@ describe('Complex Queue Scenarios', () => {
|
|
|
189
189
|
await archiveQueue.clear();
|
|
190
190
|
});
|
|
191
191
|
it('should prune expired pending tasks', async () => {
|
|
192
|
-
// Time to live:
|
|
193
|
-
const task = await queue.enqueue('expire-me', {}, { timeToLive: currentTimestamp() +
|
|
194
|
-
await timeout(
|
|
192
|
+
// Time to live: 50ms
|
|
193
|
+
const task = await queue.enqueue('expire-me', {}, { timeToLive: currentTimestamp() + 50 });
|
|
194
|
+
await timeout(60);
|
|
195
195
|
await queue.maintenance();
|
|
196
196
|
const updated = await queue.getTask(task.id);
|
|
197
197
|
expect(updated?.status).toBe(TaskStatus.Dead);
|
|
@@ -206,7 +206,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
206
206
|
const d = await queue.dequeue();
|
|
207
207
|
await queue.complete(d);
|
|
208
208
|
// Force move
|
|
209
|
-
await timeout(
|
|
209
|
+
await timeout(60);
|
|
210
210
|
await queue.maintenance();
|
|
211
211
|
// Verify retrieval
|
|
212
212
|
const fromArchive = await queue.getTask(task.id);
|
|
@@ -215,7 +215,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
215
215
|
});
|
|
216
216
|
it('should defer archival of parent tasks until children are archived', async () => {
|
|
217
217
|
const qProvider = injector.resolve(TaskQueueProvider);
|
|
218
|
-
const treeQueue = qProvider.get(`archive-tree-${Date.now()}`, { retention:
|
|
218
|
+
const treeQueue = qProvider.get(`archive-tree-${Date.now()}`, { retention: 50 });
|
|
219
219
|
const parent = await treeQueue.enqueue('parent', {});
|
|
220
220
|
const child = await treeQueue.enqueue('child', {}, { parentId: parent.id });
|
|
221
221
|
const d1 = await treeQueue.dequeue();
|
|
@@ -223,7 +223,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
223
223
|
await treeQueue.complete(d1);
|
|
224
224
|
await treeQueue.complete(d2);
|
|
225
225
|
// Wait for retention
|
|
226
|
-
await timeout(
|
|
226
|
+
await timeout(60);
|
|
227
227
|
// First maintenance: should archive child, but parent stays because child is still in main table (until it's deleted in the same tx maybe? No, loadMany happens before delete)
|
|
228
228
|
await treeQueue.maintenance();
|
|
229
229
|
const parentStillActive = await treeQueue.getTask(parent.id);
|
|
@@ -282,7 +282,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
282
282
|
for (let i = 0; i < 5; i++) {
|
|
283
283
|
if (u?.status == TaskStatus.Waiting)
|
|
284
284
|
break;
|
|
285
|
-
await timeout(
|
|
285
|
+
await timeout(10);
|
|
286
286
|
u = await queue.getTask(dependent.id);
|
|
287
287
|
}
|
|
288
288
|
expect(u?.status).toBe(TaskStatus.Waiting); // Should still be waiting because dependency didn't Complete
|