@ardrive/turbo-sdk 1.42.0-alpha.13 → 1.42.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/cli/commands/arns.js +8 -3
- package/lib/cjs/common/folderIndex.js +394 -0
- package/lib/cjs/common/index.js +1 -0
- package/lib/cjs/common/turbo.js +11 -4
- package/lib/cjs/common/upload.js +288 -17
- package/lib/cjs/node/folderIndex.js +132 -0
- package/lib/cjs/node/index.js +1 -0
- package/lib/cjs/node/upload.js +7 -0
- package/lib/cjs/utils/folderIndex.js +88 -0
- package/lib/esm/cli/commands/arns.js +8 -3
- package/lib/esm/common/folderIndex.js +389 -0
- package/lib/esm/common/index.js +1 -0
- package/lib/esm/common/turbo.js +11 -4
- package/lib/esm/common/upload.js +289 -18
- package/lib/esm/node/folderIndex.js +129 -0
- package/lib/esm/node/index.js +1 -0
- package/lib/esm/node/upload.js +7 -0
- package/lib/esm/utils/folderIndex.js +81 -0
- package/lib/types/cli/commands/arns.d.ts.map +1 -1
- package/lib/types/common/folderIndex.d.ts +54 -0
- package/lib/types/common/folderIndex.d.ts.map +1 -0
- package/lib/types/common/index.d.ts +1 -0
- package/lib/types/common/index.d.ts.map +1 -1
- package/lib/types/common/turbo.d.ts +11 -4
- package/lib/types/common/turbo.d.ts.map +1 -1
- package/lib/types/common/upload.d.ts +41 -0
- package/lib/types/common/upload.d.ts.map +1 -1
- package/lib/types/node/folderIndex.d.ts +28 -0
- package/lib/types/node/folderIndex.d.ts.map +1 -0
- package/lib/types/node/index.d.ts +1 -0
- package/lib/types/node/index.d.ts.map +1 -1
- package/lib/types/node/upload.d.ts +1 -0
- package/lib/types/node/upload.d.ts.map +1 -1
- package/lib/types/types.d.ts +163 -0
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/utils/folderIndex.d.ts +47 -0
- package/lib/types/utils/folderIndex.d.ts.map +1 -0
- package/package.json +1 -1
package/lib/esm/common/upload.js
CHANGED
|
@@ -16,9 +16,10 @@
|
|
|
16
16
|
import { BigNumber } from 'bignumber.js';
|
|
17
17
|
import { pLimit } from 'plimit-lit';
|
|
18
18
|
import { ExistingBalanceFunding, OnDemandFunding, X402Funding, } from '../types.js';
|
|
19
|
-
import { isBlob, sleep } from '../utils/common.js';
|
|
19
|
+
import { isBlob, isValidArweaveBase64URL, sleep } from '../utils/common.js';
|
|
20
20
|
import { AbortError } from '../utils/errors.js';
|
|
21
21
|
import { FailedRequestError } from '../utils/errors.js';
|
|
22
|
+
import { contentHashFromFolderIndexKey, contentHashTagName, folderIndexKey, } from '../utils/folderIndex.js';
|
|
22
23
|
import { ChunkedUploader } from './chunked.js';
|
|
23
24
|
import { TurboEventEmitter, createStreamWithUploadEvents } from './events.js';
|
|
24
25
|
import { defaultRetryConfig } from './http.js';
|
|
@@ -366,6 +367,214 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
366
367
|
};
|
|
367
368
|
return manifest;
|
|
368
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* The sha-256 of a file's bytes, as lowercase hex. The key of a
|
|
372
|
+
* {@link TurboFolderUploadIndex}.
|
|
373
|
+
*
|
|
374
|
+
* This default consumes the file's stream and digests it with the platform
|
|
375
|
+
* WebCrypto implementation, which serves the browser. NodeJS overrides it
|
|
376
|
+
* with a streaming digest so a large file is never held in memory.
|
|
377
|
+
*/
|
|
378
|
+
async computeContentHash(file) {
|
|
379
|
+
const stream = this.getFileStreamForFile(file);
|
|
380
|
+
const chunks = [];
|
|
381
|
+
if ('getReader' in stream) {
|
|
382
|
+
const reader = stream.getReader();
|
|
383
|
+
for (;;) {
|
|
384
|
+
const { done, value } = await reader.read();
|
|
385
|
+
if (done)
|
|
386
|
+
break;
|
|
387
|
+
chunks.push(value);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
else {
|
|
391
|
+
for await (const chunk of stream) {
|
|
392
|
+
chunks.push(new Uint8Array(chunk));
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
const digest = await crypto.subtle.digest('SHA-256', Buffer.concat(chunks));
|
|
396
|
+
return Buffer.from(digest).toString('hex');
|
|
397
|
+
}
|
|
398
|
+
/**
|
|
399
|
+
* The exact tag set a folder upload writes for one file. Shared by the
|
|
400
|
+
* planner and the uploader, so that the tags a folder index key is computed
|
|
401
|
+
* from are, without question, the tags the data item ends up carrying.
|
|
402
|
+
*/
|
|
403
|
+
folderFileTags({ file, dataItemOpts, contentHash, hashTagName = contentHashTagName, }) {
|
|
404
|
+
return [
|
|
405
|
+
...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type' &&
|
|
406
|
+
// Only stripped when this upload is going to write its own. Without a
|
|
407
|
+
// folder index, a caller's File-SHA256 tag is theirs to keep.
|
|
408
|
+
(contentHash === undefined || tag.name !== hashTagName)) ?? []),
|
|
409
|
+
{
|
|
410
|
+
name: 'Content-Type',
|
|
411
|
+
value: this.getContentType(file, dataItemOpts),
|
|
412
|
+
},
|
|
413
|
+
...(contentHash !== undefined
|
|
414
|
+
? [{ name: hashTagName, value: contentHash }]
|
|
415
|
+
: []),
|
|
416
|
+
];
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* A folder index is a cache, so a layer that is unreachable must mean a miss
|
|
420
|
+
* and not an aborted deploy. The write side is forgiving for the same reason.
|
|
421
|
+
*/
|
|
422
|
+
async readFolderIndex(folderIndex, keys, signal) {
|
|
423
|
+
const knownIds = new Map();
|
|
424
|
+
const unresolved = [];
|
|
425
|
+
for (const key of keys) {
|
|
426
|
+
let id;
|
|
427
|
+
try {
|
|
428
|
+
id = await folderIndex.get(key);
|
|
429
|
+
}
|
|
430
|
+
catch (error) {
|
|
431
|
+
this.logger.error('Failed to read from folder index', error);
|
|
432
|
+
}
|
|
433
|
+
if (id !== undefined && isValidArweaveBase64URL(id)) {
|
|
434
|
+
knownIds.set(key, id);
|
|
435
|
+
}
|
|
436
|
+
else {
|
|
437
|
+
unresolved.push(key);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
if (unresolved.length > 0 && folderIndex.resolve !== undefined) {
|
|
441
|
+
try {
|
|
442
|
+
const resolved = await folderIndex.resolve(unresolved, { signal });
|
|
443
|
+
for (const [key, id] of Object.entries(resolved)) {
|
|
444
|
+
if (isValidArweaveBase64URL(id)) {
|
|
445
|
+
knownIds.set(key, id);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
catch (error) {
|
|
450
|
+
this.logger.error('Failed to resolve from folder index', error);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return knownIds;
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Hashes every file and derives its folder index key, then works out which of
|
|
457
|
+
* those keys already have a data item id -- from the index, or from an
|
|
458
|
+
* identical file earlier in this same folder.
|
|
459
|
+
*/
|
|
460
|
+
async planFolderIndex({ files, folderIndex, dataItemOpts, limit, signal, }) {
|
|
461
|
+
const contentHashes = new Map();
|
|
462
|
+
const itemKeys = new Map();
|
|
463
|
+
await Promise.all(files.map((file) => limit(async () => {
|
|
464
|
+
let contentHash;
|
|
465
|
+
try {
|
|
466
|
+
contentHash = await this.computeContentHash(file);
|
|
467
|
+
}
|
|
468
|
+
catch (error) {
|
|
469
|
+
// A file that cannot be read is left un-keyed and falls through to
|
|
470
|
+
// the per-file upload path, which reports it through `errors` under
|
|
471
|
+
// `throwOnFailure`. Rejecting here instead would abort the whole
|
|
472
|
+
// folder before any upload starts, so passing a folder index would
|
|
473
|
+
// silently change the failure contract.
|
|
474
|
+
this.logger.error('Failed to hash file for the folder index', {
|
|
475
|
+
file: this.getFileName(file),
|
|
476
|
+
error,
|
|
477
|
+
});
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
contentHashes.set(file, contentHash);
|
|
481
|
+
itemKeys.set(file, await folderIndexKey({
|
|
482
|
+
contentHash,
|
|
483
|
+
tags: this.folderFileTags({
|
|
484
|
+
file,
|
|
485
|
+
dataItemOpts,
|
|
486
|
+
contentHash,
|
|
487
|
+
hashTagName: folderIndex.hashTagName,
|
|
488
|
+
}),
|
|
489
|
+
}));
|
|
490
|
+
})));
|
|
491
|
+
const knownIds = await this.readFolderIndex(folderIndex, [...new Set(itemKeys.values())], signal);
|
|
492
|
+
// Two files that would produce the same data item share a single upload, so
|
|
493
|
+
// the plan deduplicates within the folder as well as against the index.
|
|
494
|
+
const claimed = new Set();
|
|
495
|
+
const filesToUpload = files.filter((file) => {
|
|
496
|
+
const key = itemKeys.get(file);
|
|
497
|
+
// An un-keyed file could not be hashed. It is never a reuse candidate and
|
|
498
|
+
// must not share the dedup slot with the next un-keyed file, so it skips
|
|
499
|
+
// the claimed set entirely and always uploads.
|
|
500
|
+
if (key === undefined) {
|
|
501
|
+
return true;
|
|
502
|
+
}
|
|
503
|
+
if (knownIds.has(key) || claimed.has(key)) {
|
|
504
|
+
return false;
|
|
505
|
+
}
|
|
506
|
+
claimed.add(key);
|
|
507
|
+
return true;
|
|
508
|
+
});
|
|
509
|
+
this.logger.debug('Planned folder index', {
|
|
510
|
+
folderIndex: folderIndex.name,
|
|
511
|
+
totalFiles: files.length,
|
|
512
|
+
filesToUpload: filesToUpload.length,
|
|
513
|
+
});
|
|
514
|
+
await this.warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds });
|
|
515
|
+
return { contentHashes, itemKeys, knownIds, filesToUpload };
|
|
516
|
+
}
|
|
517
|
+
/**
|
|
518
|
+
* A key covers the tags on a file as well as its bytes, so a tag in
|
|
519
|
+
* `dataItemOpts` that changes between deploys re-uploads the whole folder at
|
|
520
|
+
* full price. That is the right answer -- a reused data item is never one
|
|
521
|
+
* this call would not have made -- but on its own it is a silent cost cliff:
|
|
522
|
+
* a successful deploy, a full bill, and nothing saying why.
|
|
523
|
+
*
|
|
524
|
+
* The signature of that mistake is exact, and it is already in hand. A key is
|
|
525
|
+
* `<bytes>.<tags>`, so a file whose *bytes half* the index knows under some
|
|
526
|
+
* other tags half is a file whose content is already paid for and whose tags
|
|
527
|
+
* moved. Nothing else produces that: a folder the index has never seen has
|
|
528
|
+
* unknown bytes, and a layer that could not be reached reports nothing known.
|
|
529
|
+
* It also catches one file in a hundred, not just all of them.
|
|
530
|
+
*/
|
|
531
|
+
async warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds, }) {
|
|
532
|
+
if (folderIndex.knownContentHashes === undefined) {
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
// Counted in files, not in keys: the message is about a bill, and the bill
|
|
536
|
+
// is per file. Two files that share a key are one upload but the reader is
|
|
537
|
+
// looking for their own file count.
|
|
538
|
+
const filesPerKey = new Map();
|
|
539
|
+
for (const key of itemKeys.values()) {
|
|
540
|
+
filesPerKey.set(key, (filesPerKey.get(key) ?? 0) + 1);
|
|
541
|
+
}
|
|
542
|
+
const missed = [...filesPerKey.keys()].filter((key) => !knownIds.has(key));
|
|
543
|
+
if (missed.length === 0) {
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
const filesFor = (keys) => keys.reduce((count, key) => count + (filesPerKey.get(key) ?? 0), 0);
|
|
547
|
+
const missedFiles = filesFor(missed);
|
|
548
|
+
const missedByContentHash = new Map();
|
|
549
|
+
for (const key of missed) {
|
|
550
|
+
const contentHash = contentHashFromFolderIndexKey(key);
|
|
551
|
+
missedByContentHash.set(contentHash, [
|
|
552
|
+
...(missedByContentHash.get(contentHash) ?? []),
|
|
553
|
+
key,
|
|
554
|
+
]);
|
|
555
|
+
}
|
|
556
|
+
let staleTagged = [];
|
|
557
|
+
try {
|
|
558
|
+
staleTagged =
|
|
559
|
+
(await folderIndex.knownContentHashes([
|
|
560
|
+
...missedByContentHash.keys(),
|
|
561
|
+
])) ?? [];
|
|
562
|
+
}
|
|
563
|
+
catch (error) {
|
|
564
|
+
this.logger.error('Failed to read from folder index', error);
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
const affected = staleTagged.reduce((count, contentHash) => count + filesFor(missedByContentHash.get(contentHash) ?? []), 0);
|
|
568
|
+
if (affected === 0) {
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
571
|
+
this.logger.warn(`${affected} of the ${missedFiles} file(s) this run is about to upload are already on Arweave byte for byte, under a different set of tags. ` +
|
|
572
|
+
'Their content has not changed but their tags have, so they are being paid for again. A folder index key covers the tags ' +
|
|
573
|
+
'on a file as well as its bytes. That is usually a tag in dataItemOpts whose value changes between deploys -- a commit ' +
|
|
574
|
+
'sha, a build number, a timestamp -- in which case move it to manifestDataItemOpts rather than paying for these files ' +
|
|
575
|
+
'again. It can also be a file that kept its content but changed its Content-Type, through a rename or a new extension, ' +
|
|
576
|
+
'which is expected and costs one upload.');
|
|
577
|
+
}
|
|
369
578
|
getContentType(file, dataItemOpts) {
|
|
370
579
|
const userDefinedContentType = dataItemOpts?.tags?.find((tag) => tag.name === 'Content-Type')?.value;
|
|
371
580
|
if (userDefinedContentType !== undefined) {
|
|
@@ -375,7 +584,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
375
584
|
}
|
|
376
585
|
async uploadFolder(params) {
|
|
377
586
|
this.logger.debug('Uploading folder...', { params });
|
|
378
|
-
const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
|
|
587
|
+
const { dataItemOpts, manifestDataItemOpts = dataItemOpts, folderIndex, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
|
|
379
588
|
const { disableManifest, indexFile, fallbackFile } = manifestOptions;
|
|
380
589
|
// Create event emitter from events parameter
|
|
381
590
|
const emitter = new TurboEventEmitter(events);
|
|
@@ -384,15 +593,37 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
384
593
|
fileResponses: [],
|
|
385
594
|
};
|
|
386
595
|
const errors = [];
|
|
596
|
+
const limit = pLimit(maxConcurrentUploads);
|
|
387
597
|
// Get files and calculate total bytes upfront for progress tracking
|
|
388
598
|
const files = await this.getFiles(params);
|
|
389
|
-
|
|
599
|
+
// With an index, only the files whose bytes are not already on Arweave are
|
|
600
|
+
// signed, uploaded and paid for. Progress totals cover that subset, since
|
|
601
|
+
// it is all this call actually does.
|
|
602
|
+
let contentHashes = new Map();
|
|
603
|
+
let itemKeys = new Map();
|
|
604
|
+
let knownIds = new Map();
|
|
605
|
+
let filesToUpload = files;
|
|
606
|
+
if (folderIndex !== undefined) {
|
|
607
|
+
({ contentHashes, itemKeys, knownIds, filesToUpload } =
|
|
608
|
+
await this.planFolderIndex({
|
|
609
|
+
files,
|
|
610
|
+
folderIndex,
|
|
611
|
+
dataItemOpts,
|
|
612
|
+
limit,
|
|
613
|
+
signal,
|
|
614
|
+
}));
|
|
615
|
+
}
|
|
616
|
+
const totalFiles = filesToUpload.length;
|
|
390
617
|
let totalBytes = 0;
|
|
618
|
+
let folderBytes = 0;
|
|
391
619
|
const fileSizes = new Map();
|
|
392
620
|
files.forEach((file) => {
|
|
393
621
|
const size = this.getFileSize(file);
|
|
394
622
|
fileSizes.set(file, size);
|
|
395
|
-
|
|
623
|
+
folderBytes += size;
|
|
624
|
+
});
|
|
625
|
+
filesToUpload.forEach((file) => {
|
|
626
|
+
totalBytes += fileSizes.get(file) ?? 0;
|
|
396
627
|
});
|
|
397
628
|
// Track progress across all files
|
|
398
629
|
let processedFiles = 0;
|
|
@@ -407,13 +638,15 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
407
638
|
fileIndex,
|
|
408
639
|
totalFiles,
|
|
409
640
|
});
|
|
410
|
-
const
|
|
641
|
+
const contentHash = contentHashes.get(file);
|
|
411
642
|
const dataItemOptsWithContentType = {
|
|
412
643
|
...dataItemOpts,
|
|
413
|
-
tags:
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
644
|
+
tags: this.folderFileTags({
|
|
645
|
+
file,
|
|
646
|
+
dataItemOpts,
|
|
647
|
+
contentHash,
|
|
648
|
+
hashTagName: folderIndex?.hashTagName,
|
|
649
|
+
}),
|
|
417
650
|
};
|
|
418
651
|
try {
|
|
419
652
|
const result = await this.uploadFile({
|
|
@@ -458,8 +691,24 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
458
691
|
},
|
|
459
692
|
fundingMode,
|
|
460
693
|
});
|
|
461
|
-
const
|
|
462
|
-
|
|
694
|
+
const itemKey = itemKeys.get(file);
|
|
695
|
+
if (itemKey !== undefined) {
|
|
696
|
+
// Recorded before anything else can fail, so a run that is killed
|
|
697
|
+
// part way through never loses a file it has already paid for.
|
|
698
|
+
knownIds.set(itemKey, result.id);
|
|
699
|
+
try {
|
|
700
|
+
await folderIndex?.set(itemKey, result.id);
|
|
701
|
+
}
|
|
702
|
+
catch (error) {
|
|
703
|
+
// A failed index write costs the next run a re-upload; it must not
|
|
704
|
+
// fail this one, which has already paid for and landed the bytes.
|
|
705
|
+
this.logger.error('Failed to write to folder index', error);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
else {
|
|
709
|
+
const relativePath = this.getRelativePath(file, params);
|
|
710
|
+
paths[relativePath] = { id: result.id };
|
|
711
|
+
}
|
|
463
712
|
response.fileResponses.push(result);
|
|
464
713
|
// Update processed counts after file completes
|
|
465
714
|
processedFiles++;
|
|
@@ -495,10 +744,9 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
495
744
|
errors.push(error);
|
|
496
745
|
}
|
|
497
746
|
};
|
|
498
|
-
const limit = pLimit(maxConcurrentUploads);
|
|
499
747
|
let cryptoFundResult;
|
|
500
748
|
if (fundingMode instanceof OnDemandFunding) {
|
|
501
|
-
const totalByteCount =
|
|
749
|
+
const totalByteCount = filesToUpload.reduce((acc, file) => {
|
|
502
750
|
return acc + this.getFileSize(file) + 1200; // allow extra per file for ANS-104 headers
|
|
503
751
|
}, 0);
|
|
504
752
|
cryptoFundResult = await this.onDemand({
|
|
@@ -506,15 +754,36 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
506
754
|
onDemandFunding: fundingMode,
|
|
507
755
|
});
|
|
508
756
|
}
|
|
509
|
-
await Promise.all(
|
|
757
|
+
await Promise.all(filesToUpload.map((file, index) => limit(() => uploadFile(file, index))));
|
|
510
758
|
this.logger.debug('Finished uploading files', {
|
|
511
|
-
numFiles:
|
|
759
|
+
numFiles: filesToUpload.length,
|
|
512
760
|
numErrors: errors.length,
|
|
513
761
|
results: response.fileResponses,
|
|
514
762
|
});
|
|
515
763
|
if (errors.length > 0) {
|
|
516
764
|
response.errors = errors;
|
|
517
765
|
}
|
|
766
|
+
if (folderIndex !== undefined) {
|
|
767
|
+
// Built in folder order rather than in completion order, so an unchanged
|
|
768
|
+
// folder produces byte identical manifest bytes -- and therefore an
|
|
769
|
+
// identical manifest data item id -- run after run.
|
|
770
|
+
for (const file of files) {
|
|
771
|
+
const id = knownIds.get(itemKeys.get(file));
|
|
772
|
+
if (id !== undefined) {
|
|
773
|
+
paths[this.getRelativePath(file, params)] = { id };
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
response.folderIndexSummary = {
|
|
777
|
+
totalFiles: files.length,
|
|
778
|
+
totalBytes: folderBytes,
|
|
779
|
+
// What landed, not what was planned. With throwOnFailure: false a file
|
|
780
|
+
// can be planned, paid for nothing, and never arrive.
|
|
781
|
+
uploadedFiles: processedFiles,
|
|
782
|
+
uploadedBytes: processedBytes,
|
|
783
|
+
reusedFiles: files.length - filesToUpload.length,
|
|
784
|
+
reusedBytes: folderBytes - totalBytes,
|
|
785
|
+
};
|
|
786
|
+
}
|
|
518
787
|
if (disableManifest) {
|
|
519
788
|
emitter.emit('folder-success');
|
|
520
789
|
return response;
|
|
@@ -533,8 +802,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
533
802
|
fallbackFile,
|
|
534
803
|
});
|
|
535
804
|
const tagsWithManifestContentType = [
|
|
536
|
-
...(
|
|
537
|
-
[]),
|
|
805
|
+
...(manifestDataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ?? []),
|
|
538
806
|
{ name: 'Content-Type', value: 'application/x.arweave-manifest+json' },
|
|
539
807
|
];
|
|
540
808
|
const manifestBuffer = Buffer.from(JSON.stringify(manifest));
|
|
@@ -544,7 +812,10 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
|
|
|
544
812
|
fileStreamFactory: () => this.createManifestStream(manifestBuffer),
|
|
545
813
|
fileSizeFactory: () => manifestBuffer.byteLength,
|
|
546
814
|
signal,
|
|
547
|
-
dataItemOpts: {
|
|
815
|
+
dataItemOpts: {
|
|
816
|
+
...manifestDataItemOpts,
|
|
817
|
+
tags: tagsWithManifestContentType,
|
|
818
|
+
},
|
|
548
819
|
chunkByteCount,
|
|
549
820
|
maxChunkConcurrency,
|
|
550
821
|
maxFinalizeMs,
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync, } from 'node:fs';
|
|
17
|
+
import { dirname, resolve as resolvePath } from 'node:path';
|
|
18
|
+
import { isValidArweaveBase64URL } from '../utils/common.js';
|
|
19
|
+
import { contentHashFromFolderIndexKey, isValidFolderIndexKey, } from '../utils/folderIndex.js';
|
|
20
|
+
const fileFolderIndexVersion = 2;
|
|
21
|
+
/**
|
|
22
|
+
* A local record of folder index key to data item id, for uploads on a machine
|
|
23
|
+
* that keeps its working directory between deploys.
|
|
24
|
+
*
|
|
25
|
+
* The file is an append-only log, one JSON record per line, compacted when it
|
|
26
|
+
* is loaded. That shape falls out of two requirements pulling against each
|
|
27
|
+
* other:
|
|
28
|
+
*
|
|
29
|
+
* - It has to be written after every single upload, not once at the end of the
|
|
30
|
+
* run. An upload that has been paid for but forgotten is money burnt, and a
|
|
31
|
+
* deploy killed part way through is the normal case rather than the
|
|
32
|
+
* exceptional one.
|
|
33
|
+
* - Rewriting the whole file per upload is quadratic. A first deploy or a
|
|
34
|
+
* migration of a few thousand files spends minutes and gigabytes of writes on
|
|
35
|
+
* nothing but bookkeeping.
|
|
36
|
+
*
|
|
37
|
+
* Appending one line is constant work, and it is *more* crash safe than a
|
|
38
|
+
* rewrite rather than less: a process killed mid write can only damage the last
|
|
39
|
+
* line, which is dropped on load, where a torn rewrite loses every id in the
|
|
40
|
+
* file. Compaction happens once per load, through a temp file and a rename, so
|
|
41
|
+
* the readable file is never the partial one.
|
|
42
|
+
*
|
|
43
|
+
* Malformed records are dropped. A hand edited index must never be able to
|
|
44
|
+
* publish a manifest pointing at nothing.
|
|
45
|
+
*/
|
|
46
|
+
export function createFileFolderIndex({ filePath, logger, }) {
|
|
47
|
+
const absolutePath = resolvePath(filePath);
|
|
48
|
+
const tempPath = `${absolutePath}.tmp`;
|
|
49
|
+
const map = new Map();
|
|
50
|
+
const record = (key, id) => `${JSON.stringify({ k: key, i: id })}\n`;
|
|
51
|
+
const header = `${JSON.stringify({ folderIndex: fileFolderIndexVersion })}\n`;
|
|
52
|
+
/** Rewrites the log as one record per live entry, atomically. */
|
|
53
|
+
const compact = () => {
|
|
54
|
+
mkdirSync(dirname(absolutePath), { recursive: true });
|
|
55
|
+
const body = [...map]
|
|
56
|
+
// Code units, not `localeCompare`, for the same reason the key encoding
|
|
57
|
+
// avoids it: it is not a total order over distinct strings and depends on
|
|
58
|
+
// the host's ICU data. Only the file's readability rides on this one, but
|
|
59
|
+
// a sort that can call two distinct keys equal has no business here.
|
|
60
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
61
|
+
.map(([key, id]) => record(key, id))
|
|
62
|
+
.join('');
|
|
63
|
+
try {
|
|
64
|
+
writeFileSync(tempPath, `${header}${body}`);
|
|
65
|
+
renameSync(tempPath, absolutePath);
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
rmSync(tempPath, { force: true });
|
|
69
|
+
throw error;
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
let lineCount = 0;
|
|
73
|
+
if (existsSync(absolutePath)) {
|
|
74
|
+
let malformed = 0;
|
|
75
|
+
// A line at a time, so a truncated tail costs one entry rather than all of
|
|
76
|
+
// them.
|
|
77
|
+
for (const line of readFileSync(absolutePath, 'utf-8').split('\n')) {
|
|
78
|
+
if (line.length === 0) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
lineCount++;
|
|
82
|
+
try {
|
|
83
|
+
const parsed = JSON.parse(line);
|
|
84
|
+
if (parsed?.folderIndex !== undefined) {
|
|
85
|
+
continue; // header
|
|
86
|
+
}
|
|
87
|
+
if (isValidFolderIndexKey(parsed?.k) &&
|
|
88
|
+
isValidArweaveBase64URL(parsed?.i)) {
|
|
89
|
+
map.set(parsed.k, parsed.i); // last write wins
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
malformed++;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
malformed++;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (malformed > 0) {
|
|
100
|
+
logger?.warn(`Dropped ${malformed} unreadable record(s) from the folder index at ${absolutePath}`);
|
|
101
|
+
}
|
|
102
|
+
// Only worth rewriting when the log has actually accumulated slack.
|
|
103
|
+
if (lineCount > map.size + 1) {
|
|
104
|
+
try {
|
|
105
|
+
compact();
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
logger?.warn(`Could not compact the folder index at ${absolutePath}`, error);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
name: `file:${absolutePath}`,
|
|
114
|
+
get: (key) => map.get(key),
|
|
115
|
+
set: (key, id) => {
|
|
116
|
+
map.set(key, id);
|
|
117
|
+
mkdirSync(dirname(absolutePath), { recursive: true });
|
|
118
|
+
if (!existsSync(absolutePath)) {
|
|
119
|
+
writeFileSync(absolutePath, header);
|
|
120
|
+
}
|
|
121
|
+
appendFileSync(absolutePath, record(key, id));
|
|
122
|
+
},
|
|
123
|
+
knownContentHashes: (contentHashes) => {
|
|
124
|
+
const known = new Set([...map.keys()].map(contentHashFromFolderIndexKey));
|
|
125
|
+
return contentHashes.filter((contentHash) => known.has(contentHash));
|
|
126
|
+
},
|
|
127
|
+
entries: () => Object.fromEntries(map),
|
|
128
|
+
};
|
|
129
|
+
}
|
package/lib/esm/node/index.js
CHANGED
package/lib/esm/node/upload.js
CHANGED
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { createReadStream, promises, statSync } from 'fs';
|
|
17
17
|
import { lookup } from 'mime-types';
|
|
18
|
+
import { createHash } from 'node:crypto';
|
|
19
|
+
import { pipeline } from 'node:stream/promises';
|
|
18
20
|
import { join } from 'path';
|
|
19
21
|
import { Readable } from 'stream';
|
|
20
22
|
import { TurboAuthenticatedBaseUploadService, defaultUploadServiceURL, } from '../common/upload.js';
|
|
@@ -84,4 +86,9 @@ export class TurboAuthenticatedUploadService extends TurboAuthenticatedBaseUploa
|
|
|
84
86
|
createManifestStream(manifestBuffer) {
|
|
85
87
|
return Readable.from(manifestBuffer);
|
|
86
88
|
}
|
|
89
|
+
async computeContentHash(file) {
|
|
90
|
+
const hash = createHash('sha256');
|
|
91
|
+
await pipeline(createReadStream(file), hash);
|
|
92
|
+
return hash.digest('hex');
|
|
93
|
+
}
|
|
87
94
|
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Internals shared by the folder index layers and by `uploadFolder`. Not part
|
|
18
|
+
* of the package's public surface.
|
|
19
|
+
*/
|
|
20
|
+
/** Tag carrying the sha-256 of a file's own bytes, as lowercase hex. */
|
|
21
|
+
export const contentHashTagName = 'File-SHA256';
|
|
22
|
+
const contentHashRegex = /^[0-9a-f]{64}$/;
|
|
23
|
+
const folderIndexKeyRegex = /^[0-9a-f]{64}\.[0-9a-f]{64}$/;
|
|
24
|
+
/** True for a lowercase hex sha-256 digest of a file's bytes. */
|
|
25
|
+
export function isValidContentHash(hash) {
|
|
26
|
+
return hash !== undefined && contentHashRegex.test(hash);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* A folder index key is `<sha-256 of the bytes>.<sha-256 of the tags>`.
|
|
30
|
+
*
|
|
31
|
+
* Both halves matter. Keying on the bytes alone would reuse an item whose tags
|
|
32
|
+
* are not the ones the caller asked for: an empty `a.css` and an empty `b.js`
|
|
33
|
+
* hash identically, and reusing one for the other would serve JavaScript as
|
|
34
|
+
* `text/css`. Keying on the tags as well means a reused data item is always
|
|
35
|
+
* exactly the data item this call would otherwise have created.
|
|
36
|
+
*
|
|
37
|
+
* The bytes half is kept in the clear so a layer that queries a gateway can
|
|
38
|
+
* recover it and filter on the `File-SHA256` tag.
|
|
39
|
+
*/
|
|
40
|
+
export function isValidFolderIndexKey(key) {
|
|
41
|
+
return key !== undefined && folderIndexKeyRegex.test(key);
|
|
42
|
+
}
|
|
43
|
+
/** The bytes half of a folder index key. */
|
|
44
|
+
export function contentHashFromFolderIndexKey(key) {
|
|
45
|
+
return key.slice(0, 64);
|
|
46
|
+
}
|
|
47
|
+
async function sha256Hex(data) {
|
|
48
|
+
const digest = await crypto.subtle.digest('SHA-256', data);
|
|
49
|
+
return Buffer.from(digest).toString('hex');
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Canonical encoding of a tag set: the same tags in a different order encode
|
|
53
|
+
* identically, because two data items that differ only in tag order are the
|
|
54
|
+
* same item as far as reuse goes.
|
|
55
|
+
*
|
|
56
|
+
* The sort compares code units rather than using `localeCompare`, which is not
|
|
57
|
+
* a total order over distinct strings and depends on the host's ICU data --
|
|
58
|
+
* NFC and NFD spellings of the same word compare equal without being equal, and
|
|
59
|
+
* two locales disagree on where an accented letter sorts. Either would let one
|
|
60
|
+
* tag set hash differently on two machines, which costs a reuse.
|
|
61
|
+
*
|
|
62
|
+
* The encoding has to be injective, not merely stable. An ANS-104 tag name and
|
|
63
|
+
* value are arbitrary byte arrays, so a delimiter-joined string collides: with
|
|
64
|
+
* a 0x00/0x01 join, a single tag whose value embeds both delimiters encodes
|
|
65
|
+
* exactly like two separate tags. A collision here is a wrong reuse, which is
|
|
66
|
+
* the one failure the composite key exists to make impossible. JSON escapes
|
|
67
|
+
* the delimiters, so no value can forge a record boundary.
|
|
68
|
+
*/
|
|
69
|
+
function canonicalTags(tags) {
|
|
70
|
+
const pairs = tags
|
|
71
|
+
.map((tag) => [tag.name, tag.value])
|
|
72
|
+
.sort(([aName, aValue], [bName, bValue]) => {
|
|
73
|
+
const [a, b] = aName === bName ? [aValue, bValue] : [aName, bName];
|
|
74
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
75
|
+
});
|
|
76
|
+
return new TextEncoder().encode(JSON.stringify(pairs));
|
|
77
|
+
}
|
|
78
|
+
/** The folder index key for a file with these bytes and these final tags. */
|
|
79
|
+
export async function folderIndexKey({ contentHash, tags, }) {
|
|
80
|
+
return `${contentHash}.${await sha256Hex(canonicalTags(tags))}`;
|
|
81
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arns.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/arns.ts"],"names":[],"mappings":"AAmBA,OAAO,EACL,UAAU,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,6BAA6B,EAC7B,YAAY,EACZ,eAAe,EACf,iBAAiB,EACjB,0BAA0B,EAG3B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAMtE,OAAO,EACL,wBAAwB,EACxB,sBAAsB,EACtB,oBAAoB,EACpB,gBAAgB,EAChB,mBAAmB,EACnB,yBAAyB,EACzB,2BAA2B,EAC3B,+BAA+B,EAC/B,uBAAuB,EACvB,4BAA4B,EAC5B,oBAAoB,EACpB,sBAAsB,EACtB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAyBrB,MAAM,MAAM,eAAe,GAAG;IAC5B,mBAAmB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC1E,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qBAAqB,CAAC,CAAC,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;CACzC,CAAC;AACF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,WAAW,CAAC,MAAM,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,eAAe,CAAC;QACvB,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,0BAA0B,CAAC,MAAM,EAAE;QACjC,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AACF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,eAAe,CAAC,MAAM,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,aAAa,CAAC,MAAM,EAAE;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,gBAAgB,CAAC,MAAM,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,iBAAiB,CAAC,MAAM,EAAE;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,oBAAoB,CAAC,MAAM,EAAE;QAC3B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,qBAAqB,CAAC,MAAM,EAAE;QAC5B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAClC,cAAc,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;KAClC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,wBAAwB,CAAC,MAAM,EAAE;QAC/B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,kBAAkB,CAAC,MAAM,EAAE;QACzB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AAUF,MAAM,MAAM,qBAAqB,GAAG;IAClC,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;CAC1E,CAAC;AAEF,wBAAgB,uBAAuB,CAAC,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAK1E;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,IAAI,EAAE,MAAM,GACX,MAAM,CASR;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAKvE;AAED,sFAAsF;AACtF,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,UAAU,CAiB1E;AAUD,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,GAC3B,MAAM,EAAE,GAAG,SAAS,CAEtB;AASD;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,gBAAgB,GACxB,eAAe,CAwCjB;AAyCD,wBAAsB,SAAS,CAC7B,OAAO,EAAE,gBAAgB,EACzB,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"arns.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/arns.ts"],"names":[],"mappings":"AAmBA,OAAO,EACL,UAAU,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,6BAA6B,EAC7B,YAAY,EACZ,eAAe,EACf,iBAAiB,EACjB,0BAA0B,EAG3B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAMtE,OAAO,EACL,wBAAwB,EACxB,sBAAsB,EACtB,oBAAoB,EACpB,gBAAgB,EAChB,mBAAmB,EACnB,yBAAyB,EACzB,2BAA2B,EAC3B,+BAA+B,EAC/B,uBAAuB,EACvB,4BAA4B,EAC5B,oBAAoB,EACpB,sBAAsB,EACtB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAyBrB,MAAM,MAAM,eAAe,GAAG;IAC5B,mBAAmB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC1E,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qBAAqB,CAAC,CAAC,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;CACzC,CAAC;AACF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,WAAW,CAAC,MAAM,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,eAAe,CAAC;QACvB,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,0BAA0B,CAAC,MAAM,EAAE;QACjC,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AACF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,eAAe,CAAC,MAAM,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,aAAa,CAAC,MAAM,EAAE;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,gBAAgB,CAAC,MAAM,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,iBAAiB,CAAC,MAAM,EAAE;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,oBAAoB,CAAC,MAAM,EAAE;QAC3B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,qBAAqB,CAAC,MAAM,EAAE;QAC5B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAClC,cAAc,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;KAClC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,wBAAwB,CAAC,MAAM,EAAE;QAC/B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,kBAAkB,CAAC,MAAM,EAAE;QACzB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AAUF,MAAM,MAAM,qBAAqB,GAAG;IAClC,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;CAC1E,CAAC;AAEF,wBAAgB,uBAAuB,CAAC,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAK1E;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,IAAI,EAAE,MAAM,GACX,MAAM,CASR;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAKvE;AAED,sFAAsF;AACtF,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,UAAU,CAiB1E;AAUD,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,GAC3B,MAAM,EAAE,GAAG,SAAS,CAEtB;AASD;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,gBAAgB,GACxB,eAAe,CAwCjB;AAyCD,wBAAsB,SAAS,CAC7B,OAAO,EAAE,gBAAgB,EACzB,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,wBAAwB,CACtB,MAAM,EAAE,2BAA2B,GAClC,OAAO,CAAC,6BAA6B,CAAC,CAAC;CAC3C,CAAC;AAEF,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA4Df;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAWf;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,yBAAyB,EAClC,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,EAC/B,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAqBf;AAED,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,uBAAuB,EAChC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,wBAAwB,EACjC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,2BAA2B,EACpC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,yBAAyB,EAClC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAwBf;AAED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,4BAA4B,EACrC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAmCf;AAED,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,+BAA+B,EACxC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,EAC/B,KAAK,CAAC,EAAE,qBAAqB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAaf"}
|