@ardrive/turbo-sdk 1.42.0-alpha.13 → 1.42.0-alpha.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/lib/cjs/cli/commands/arns.js +8 -3
  2. package/lib/cjs/common/folderIndex.js +394 -0
  3. package/lib/cjs/common/index.js +1 -0
  4. package/lib/cjs/common/turbo.js +11 -4
  5. package/lib/cjs/common/upload.js +288 -17
  6. package/lib/cjs/node/folderIndex.js +132 -0
  7. package/lib/cjs/node/index.js +1 -0
  8. package/lib/cjs/node/upload.js +7 -0
  9. package/lib/cjs/utils/folderIndex.js +88 -0
  10. package/lib/esm/cli/commands/arns.js +8 -3
  11. package/lib/esm/common/folderIndex.js +389 -0
  12. package/lib/esm/common/index.js +1 -0
  13. package/lib/esm/common/turbo.js +11 -4
  14. package/lib/esm/common/upload.js +289 -18
  15. package/lib/esm/node/folderIndex.js +129 -0
  16. package/lib/esm/node/index.js +1 -0
  17. package/lib/esm/node/upload.js +7 -0
  18. package/lib/esm/utils/folderIndex.js +81 -0
  19. package/lib/types/cli/commands/arns.d.ts.map +1 -1
  20. package/lib/types/common/folderIndex.d.ts +54 -0
  21. package/lib/types/common/folderIndex.d.ts.map +1 -0
  22. package/lib/types/common/index.d.ts +1 -0
  23. package/lib/types/common/index.d.ts.map +1 -1
  24. package/lib/types/common/turbo.d.ts +11 -4
  25. package/lib/types/common/turbo.d.ts.map +1 -1
  26. package/lib/types/common/upload.d.ts +41 -0
  27. package/lib/types/common/upload.d.ts.map +1 -1
  28. package/lib/types/node/folderIndex.d.ts +28 -0
  29. package/lib/types/node/folderIndex.d.ts.map +1 -0
  30. package/lib/types/node/index.d.ts +1 -0
  31. package/lib/types/node/index.d.ts.map +1 -1
  32. package/lib/types/node/upload.d.ts +1 -0
  33. package/lib/types/node/upload.d.ts.map +1 -1
  34. package/lib/types/types.d.ts +163 -0
  35. package/lib/types/types.d.ts.map +1 -1
  36. package/lib/types/utils/folderIndex.d.ts +47 -0
  37. package/lib/types/utils/folderIndex.d.ts.map +1 -0
  38. package/package.json +1 -1
@@ -16,9 +16,10 @@
16
16
  import { BigNumber } from 'bignumber.js';
17
17
  import { pLimit } from 'plimit-lit';
18
18
  import { ExistingBalanceFunding, OnDemandFunding, X402Funding, } from '../types.js';
19
- import { isBlob, sleep } from '../utils/common.js';
19
+ import { isBlob, isValidArweaveBase64URL, sleep } from '../utils/common.js';
20
20
  import { AbortError } from '../utils/errors.js';
21
21
  import { FailedRequestError } from '../utils/errors.js';
22
+ import { contentHashFromFolderIndexKey, contentHashTagName, folderIndexKey, } from '../utils/folderIndex.js';
22
23
  import { ChunkedUploader } from './chunked.js';
23
24
  import { TurboEventEmitter, createStreamWithUploadEvents } from './events.js';
24
25
  import { defaultRetryConfig } from './http.js';
@@ -366,6 +367,214 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
366
367
  };
367
368
  return manifest;
368
369
  }
370
+ /**
371
+ * The sha-256 of a file's bytes, as lowercase hex. The key of a
372
+ * {@link TurboFolderUploadIndex}.
373
+ *
374
+ * This default consumes the file's stream and digests it with the platform
375
+ * WebCrypto implementation, which serves the browser. NodeJS overrides it
376
+ * with a streaming digest so a large file is never held in memory.
377
+ */
378
+ async computeContentHash(file) {
379
+ const stream = this.getFileStreamForFile(file);
380
+ const chunks = [];
381
+ if ('getReader' in stream) {
382
+ const reader = stream.getReader();
383
+ for (;;) {
384
+ const { done, value } = await reader.read();
385
+ if (done)
386
+ break;
387
+ chunks.push(value);
388
+ }
389
+ }
390
+ else {
391
+ for await (const chunk of stream) {
392
+ chunks.push(new Uint8Array(chunk));
393
+ }
394
+ }
395
+ const digest = await crypto.subtle.digest('SHA-256', Buffer.concat(chunks));
396
+ return Buffer.from(digest).toString('hex');
397
+ }
398
+ /**
399
+ * The exact tag set a folder upload writes for one file. Shared by the
400
+ * planner and the uploader, so that the tags a folder index key is computed
401
+ * from are, without question, the tags the data item ends up carrying.
402
+ */
403
+ folderFileTags({ file, dataItemOpts, contentHash, hashTagName = contentHashTagName, }) {
404
+ return [
405
+ ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type' &&
406
+ // Only stripped when this upload is going to write its own. Without a
407
+ // folder index, a caller's File-SHA256 tag is theirs to keep.
408
+ (contentHash === undefined || tag.name !== hashTagName)) ?? []),
409
+ {
410
+ name: 'Content-Type',
411
+ value: this.getContentType(file, dataItemOpts),
412
+ },
413
+ ...(contentHash !== undefined
414
+ ? [{ name: hashTagName, value: contentHash }]
415
+ : []),
416
+ ];
417
+ }
418
+ /**
419
+ * A folder index is a cache, so a layer that is unreachable must mean a miss
420
+ * and not an aborted deploy. The write side is forgiving for the same reason.
421
+ */
422
+ async readFolderIndex(folderIndex, keys, signal) {
423
+ const knownIds = new Map();
424
+ const unresolved = [];
425
+ for (const key of keys) {
426
+ let id;
427
+ try {
428
+ id = await folderIndex.get(key);
429
+ }
430
+ catch (error) {
431
+ this.logger.error('Failed to read from folder index', error);
432
+ }
433
+ if (id !== undefined && isValidArweaveBase64URL(id)) {
434
+ knownIds.set(key, id);
435
+ }
436
+ else {
437
+ unresolved.push(key);
438
+ }
439
+ }
440
+ if (unresolved.length > 0 && folderIndex.resolve !== undefined) {
441
+ try {
442
+ const resolved = await folderIndex.resolve(unresolved, { signal });
443
+ for (const [key, id] of Object.entries(resolved)) {
444
+ if (isValidArweaveBase64URL(id)) {
445
+ knownIds.set(key, id);
446
+ }
447
+ }
448
+ }
449
+ catch (error) {
450
+ this.logger.error('Failed to resolve from folder index', error);
451
+ }
452
+ }
453
+ return knownIds;
454
+ }
455
+ /**
456
+ * Hashes every file and derives its folder index key, then works out which of
457
+ * those keys already have a data item id -- from the index, or from an
458
+ * identical file earlier in this same folder.
459
+ */
460
+ async planFolderIndex({ files, folderIndex, dataItemOpts, limit, signal, }) {
461
+ const contentHashes = new Map();
462
+ const itemKeys = new Map();
463
+ await Promise.all(files.map((file) => limit(async () => {
464
+ let contentHash;
465
+ try {
466
+ contentHash = await this.computeContentHash(file);
467
+ }
468
+ catch (error) {
469
+ // A file that cannot be read is left un-keyed and falls through to
470
+ // the per-file upload path, which reports it through `errors` under
471
+ // `throwOnFailure`. Rejecting here instead would abort the whole
472
+ // folder before any upload starts, so passing a folder index would
473
+ // silently change the failure contract.
474
+ this.logger.error('Failed to hash file for the folder index', {
475
+ file: this.getFileName(file),
476
+ error,
477
+ });
478
+ return;
479
+ }
480
+ contentHashes.set(file, contentHash);
481
+ itemKeys.set(file, await folderIndexKey({
482
+ contentHash,
483
+ tags: this.folderFileTags({
484
+ file,
485
+ dataItemOpts,
486
+ contentHash,
487
+ hashTagName: folderIndex.hashTagName,
488
+ }),
489
+ }));
490
+ })));
491
+ const knownIds = await this.readFolderIndex(folderIndex, [...new Set(itemKeys.values())], signal);
492
+ // Two files that would produce the same data item share a single upload, so
493
+ // the plan deduplicates within the folder as well as against the index.
494
+ const claimed = new Set();
495
+ const filesToUpload = files.filter((file) => {
496
+ const key = itemKeys.get(file);
497
+ // An un-keyed file could not be hashed. It is never a reuse candidate and
498
+ // must not share the dedup slot with the next un-keyed file, so it skips
499
+ // the claimed set entirely and always uploads.
500
+ if (key === undefined) {
501
+ return true;
502
+ }
503
+ if (knownIds.has(key) || claimed.has(key)) {
504
+ return false;
505
+ }
506
+ claimed.add(key);
507
+ return true;
508
+ });
509
+ this.logger.debug('Planned folder index', {
510
+ folderIndex: folderIndex.name,
511
+ totalFiles: files.length,
512
+ filesToUpload: filesToUpload.length,
513
+ });
514
+ await this.warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds });
515
+ return { contentHashes, itemKeys, knownIds, filesToUpload };
516
+ }
517
+ /**
518
+ * A key covers the tags on a file as well as its bytes, so a tag in
519
+ * `dataItemOpts` that changes between deploys re-uploads the whole folder at
520
+ * full price. That is the right answer -- a reused data item is never one
521
+ * this call would not have made -- but on its own it is a silent cost cliff:
522
+ * a successful deploy, a full bill, and nothing saying why.
523
+ *
524
+ * The signature of that mistake is exact, and it is already in hand. A key is
525
+ * `<bytes>.<tags>`, so a file whose *bytes half* the index knows under some
526
+ * other tags half is a file whose content is already paid for and whose tags
527
+ * moved. Nothing else produces that: a folder the index has never seen has
528
+ * unknown bytes, and a layer that could not be reached reports nothing known.
529
+ * It also catches one file in a hundred, not just all of them.
530
+ */
531
+ async warnOnStaleTagMisses({ folderIndex, itemKeys, knownIds, }) {
532
+ if (folderIndex.knownContentHashes === undefined) {
533
+ return;
534
+ }
535
+ // Counted in files, not in keys: the message is about a bill, and the bill
536
+ // is per file. Two files that share a key are one upload but the reader is
537
+ // looking for their own file count.
538
+ const filesPerKey = new Map();
539
+ for (const key of itemKeys.values()) {
540
+ filesPerKey.set(key, (filesPerKey.get(key) ?? 0) + 1);
541
+ }
542
+ const missed = [...filesPerKey.keys()].filter((key) => !knownIds.has(key));
543
+ if (missed.length === 0) {
544
+ return;
545
+ }
546
+ const filesFor = (keys) => keys.reduce((count, key) => count + (filesPerKey.get(key) ?? 0), 0);
547
+ const missedFiles = filesFor(missed);
548
+ const missedByContentHash = new Map();
549
+ for (const key of missed) {
550
+ const contentHash = contentHashFromFolderIndexKey(key);
551
+ missedByContentHash.set(contentHash, [
552
+ ...(missedByContentHash.get(contentHash) ?? []),
553
+ key,
554
+ ]);
555
+ }
556
+ let staleTagged = [];
557
+ try {
558
+ staleTagged =
559
+ (await folderIndex.knownContentHashes([
560
+ ...missedByContentHash.keys(),
561
+ ])) ?? [];
562
+ }
563
+ catch (error) {
564
+ this.logger.error('Failed to read from folder index', error);
565
+ return;
566
+ }
567
+ const affected = staleTagged.reduce((count, contentHash) => count + filesFor(missedByContentHash.get(contentHash) ?? []), 0);
568
+ if (affected === 0) {
569
+ return;
570
+ }
571
+ this.logger.warn(`${affected} of the ${missedFiles} file(s) this run is about to upload are already on Arweave byte for byte, under a different set of tags. ` +
572
+ 'Their content has not changed but their tags have, so they are being paid for again. A folder index key covers the tags ' +
573
+ 'on a file as well as its bytes. That is usually a tag in dataItemOpts whose value changes between deploys -- a commit ' +
574
+ 'sha, a build number, a timestamp -- in which case move it to manifestDataItemOpts rather than paying for these files ' +
575
+ 'again. It can also be a file that kept its content but changed its Content-Type, through a rename or a new extension, ' +
576
+ 'which is expected and costs one upload.');
577
+ }
369
578
  getContentType(file, dataItemOpts) {
370
579
  const userDefinedContentType = dataItemOpts?.tags?.find((tag) => tag.name === 'Content-Type')?.value;
371
580
  if (userDefinedContentType !== undefined) {
@@ -375,7 +584,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
375
584
  }
376
585
  async uploadFolder(params) {
377
586
  this.logger.debug('Uploading folder...', { params });
378
- const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
587
+ const { dataItemOpts, manifestDataItemOpts = dataItemOpts, folderIndex, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, fundingMode = new ExistingBalanceFunding(), maxFinalizeMs, events = {}, } = params;
379
588
  const { disableManifest, indexFile, fallbackFile } = manifestOptions;
380
589
  // Create event emitter from events parameter
381
590
  const emitter = new TurboEventEmitter(events);
@@ -384,15 +593,37 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
384
593
  fileResponses: [],
385
594
  };
386
595
  const errors = [];
596
+ const limit = pLimit(maxConcurrentUploads);
387
597
  // Get files and calculate total bytes upfront for progress tracking
388
598
  const files = await this.getFiles(params);
389
- const totalFiles = files.length;
599
+ // With an index, only the files whose bytes are not already on Arweave are
600
+ // signed, uploaded and paid for. Progress totals cover that subset, since
601
+ // it is all this call actually does.
602
+ let contentHashes = new Map();
603
+ let itemKeys = new Map();
604
+ let knownIds = new Map();
605
+ let filesToUpload = files;
606
+ if (folderIndex !== undefined) {
607
+ ({ contentHashes, itemKeys, knownIds, filesToUpload } =
608
+ await this.planFolderIndex({
609
+ files,
610
+ folderIndex,
611
+ dataItemOpts,
612
+ limit,
613
+ signal,
614
+ }));
615
+ }
616
+ const totalFiles = filesToUpload.length;
390
617
  let totalBytes = 0;
618
+ let folderBytes = 0;
391
619
  const fileSizes = new Map();
392
620
  files.forEach((file) => {
393
621
  const size = this.getFileSize(file);
394
622
  fileSizes.set(file, size);
395
- totalBytes += size;
623
+ folderBytes += size;
624
+ });
625
+ filesToUpload.forEach((file) => {
626
+ totalBytes += fileSizes.get(file) ?? 0;
396
627
  });
397
628
  // Track progress across all files
398
629
  let processedFiles = 0;
@@ -407,13 +638,15 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
407
638
  fileIndex,
408
639
  totalFiles,
409
640
  });
410
- const contentType = this.getContentType(file, dataItemOpts);
641
+ const contentHash = contentHashes.get(file);
411
642
  const dataItemOptsWithContentType = {
412
643
  ...dataItemOpts,
413
- tags: [
414
- ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ?? []),
415
- { name: 'Content-Type', value: contentType },
416
- ],
644
+ tags: this.folderFileTags({
645
+ file,
646
+ dataItemOpts,
647
+ contentHash,
648
+ hashTagName: folderIndex?.hashTagName,
649
+ }),
417
650
  };
418
651
  try {
419
652
  const result = await this.uploadFile({
@@ -458,8 +691,24 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
458
691
  },
459
692
  fundingMode,
460
693
  });
461
- const relativePath = this.getRelativePath(file, params);
462
- paths[relativePath] = { id: result.id };
694
+ const itemKey = itemKeys.get(file);
695
+ if (itemKey !== undefined) {
696
+ // Recorded before anything else can fail, so a run that is killed
697
+ // part way through never loses a file it has already paid for.
698
+ knownIds.set(itemKey, result.id);
699
+ try {
700
+ await folderIndex?.set(itemKey, result.id);
701
+ }
702
+ catch (error) {
703
+ // A failed index write costs the next run a re-upload; it must not
704
+ // fail this one, which has already paid for and landed the bytes.
705
+ this.logger.error('Failed to write to folder index', error);
706
+ }
707
+ }
708
+ else {
709
+ const relativePath = this.getRelativePath(file, params);
710
+ paths[relativePath] = { id: result.id };
711
+ }
463
712
  response.fileResponses.push(result);
464
713
  // Update processed counts after file completes
465
714
  processedFiles++;
@@ -495,10 +744,9 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
495
744
  errors.push(error);
496
745
  }
497
746
  };
498
- const limit = pLimit(maxConcurrentUploads);
499
747
  let cryptoFundResult;
500
748
  if (fundingMode instanceof OnDemandFunding) {
501
- const totalByteCount = files.reduce((acc, file) => {
749
+ const totalByteCount = filesToUpload.reduce((acc, file) => {
502
750
  return acc + this.getFileSize(file) + 1200; // allow extra per file for ANS-104 headers
503
751
  }, 0);
504
752
  cryptoFundResult = await this.onDemand({
@@ -506,15 +754,36 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
506
754
  onDemandFunding: fundingMode,
507
755
  });
508
756
  }
509
- await Promise.all(files.map((file, index) => limit(() => uploadFile(file, index))));
757
+ await Promise.all(filesToUpload.map((file, index) => limit(() => uploadFile(file, index))));
510
758
  this.logger.debug('Finished uploading files', {
511
- numFiles: files.length,
759
+ numFiles: filesToUpload.length,
512
760
  numErrors: errors.length,
513
761
  results: response.fileResponses,
514
762
  });
515
763
  if (errors.length > 0) {
516
764
  response.errors = errors;
517
765
  }
766
+ if (folderIndex !== undefined) {
767
+ // Built in folder order rather than in completion order, so an unchanged
768
+ // folder produces byte identical manifest bytes -- and therefore an
769
+ // identical manifest data item id -- run after run.
770
+ for (const file of files) {
771
+ const id = knownIds.get(itemKeys.get(file));
772
+ if (id !== undefined) {
773
+ paths[this.getRelativePath(file, params)] = { id };
774
+ }
775
+ }
776
+ response.folderIndexSummary = {
777
+ totalFiles: files.length,
778
+ totalBytes: folderBytes,
779
+ // What landed, not what was planned. With throwOnFailure: false a file
780
+ // can be planned, paid for nothing, and never arrive.
781
+ uploadedFiles: processedFiles,
782
+ uploadedBytes: processedBytes,
783
+ reusedFiles: files.length - filesToUpload.length,
784
+ reusedBytes: folderBytes - totalBytes,
785
+ };
786
+ }
518
787
  if (disableManifest) {
519
788
  emitter.emit('folder-success');
520
789
  return response;
@@ -533,8 +802,7 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
533
802
  fallbackFile,
534
803
  });
535
804
  const tagsWithManifestContentType = [
536
- ...(dataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ??
537
- []),
805
+ ...(manifestDataItemOpts?.tags?.filter((tag) => tag.name !== 'Content-Type') ?? []),
538
806
  { name: 'Content-Type', value: 'application/x.arweave-manifest+json' },
539
807
  ];
540
808
  const manifestBuffer = Buffer.from(JSON.stringify(manifest));
@@ -544,7 +812,10 @@ export class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUpl
544
812
  fileStreamFactory: () => this.createManifestStream(manifestBuffer),
545
813
  fileSizeFactory: () => manifestBuffer.byteLength,
546
814
  signal,
547
- dataItemOpts: { ...dataItemOpts, tags: tagsWithManifestContentType },
815
+ dataItemOpts: {
816
+ ...manifestDataItemOpts,
817
+ tags: tagsWithManifestContentType,
818
+ },
548
819
  chunkByteCount,
549
820
  maxChunkConcurrency,
550
821
  maxFinalizeMs,
@@ -0,0 +1,129 @@
1
+ /**
2
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync, } from 'node:fs';
17
+ import { dirname, resolve as resolvePath } from 'node:path';
18
+ import { isValidArweaveBase64URL } from '../utils/common.js';
19
+ import { contentHashFromFolderIndexKey, isValidFolderIndexKey, } from '../utils/folderIndex.js';
20
+ const fileFolderIndexVersion = 2;
21
+ /**
22
+ * A local record of folder index key to data item id, for uploads on a machine
23
+ * that keeps its working directory between deploys.
24
+ *
25
+ * The file is an append-only log, one JSON record per line, compacted when it
26
+ * is loaded. That shape falls out of two requirements pulling against each
27
+ * other:
28
+ *
29
+ * - It has to be written after every single upload, not once at the end of the
30
+ * run. An upload that has been paid for but forgotten is money burnt, and a
31
+ * deploy killed part way through is the normal case rather than the
32
+ * exceptional one.
33
+ * - Rewriting the whole file per upload is quadratic. A first deploy or a
34
+ * migration of a few thousand files spends minutes and gigabytes of writes on
35
+ * nothing but bookkeeping.
36
+ *
37
+ * Appending one line is constant work, and it is *more* crash safe than a
38
+ * rewrite rather than less: a process killed mid write can only damage the last
39
+ * line, which is dropped on load, where a torn rewrite loses every id in the
40
+ * file. Compaction happens once per load, through a temp file and a rename, so
41
+ * the readable file is never the partial one.
42
+ *
43
+ * Malformed records are dropped. A hand edited index must never be able to
44
+ * publish a manifest pointing at nothing.
45
+ */
46
+ export function createFileFolderIndex({ filePath, logger, }) {
47
+ const absolutePath = resolvePath(filePath);
48
+ const tempPath = `${absolutePath}.tmp`;
49
+ const map = new Map();
50
+ const record = (key, id) => `${JSON.stringify({ k: key, i: id })}\n`;
51
+ const header = `${JSON.stringify({ folderIndex: fileFolderIndexVersion })}\n`;
52
+ /** Rewrites the log as one record per live entry, atomically. */
53
+ const compact = () => {
54
+ mkdirSync(dirname(absolutePath), { recursive: true });
55
+ const body = [...map]
56
+ // Code units, not `localeCompare`, for the same reason the key encoding
57
+ // avoids it: it is not a total order over distinct strings and depends on
58
+ // the host's ICU data. Only the file's readability rides on this one, but
59
+ // a sort that can call two distinct keys equal has no business here.
60
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
61
+ .map(([key, id]) => record(key, id))
62
+ .join('');
63
+ try {
64
+ writeFileSync(tempPath, `${header}${body}`);
65
+ renameSync(tempPath, absolutePath);
66
+ }
67
+ catch (error) {
68
+ rmSync(tempPath, { force: true });
69
+ throw error;
70
+ }
71
+ };
72
+ let lineCount = 0;
73
+ if (existsSync(absolutePath)) {
74
+ let malformed = 0;
75
+ // A line at a time, so a truncated tail costs one entry rather than all of
76
+ // them.
77
+ for (const line of readFileSync(absolutePath, 'utf-8').split('\n')) {
78
+ if (line.length === 0) {
79
+ continue;
80
+ }
81
+ lineCount++;
82
+ try {
83
+ const parsed = JSON.parse(line);
84
+ if (parsed?.folderIndex !== undefined) {
85
+ continue; // header
86
+ }
87
+ if (isValidFolderIndexKey(parsed?.k) &&
88
+ isValidArweaveBase64URL(parsed?.i)) {
89
+ map.set(parsed.k, parsed.i); // last write wins
90
+ }
91
+ else {
92
+ malformed++;
93
+ }
94
+ }
95
+ catch {
96
+ malformed++;
97
+ }
98
+ }
99
+ if (malformed > 0) {
100
+ logger?.warn(`Dropped ${malformed} unreadable record(s) from the folder index at ${absolutePath}`);
101
+ }
102
+ // Only worth rewriting when the log has actually accumulated slack.
103
+ if (lineCount > map.size + 1) {
104
+ try {
105
+ compact();
106
+ }
107
+ catch (error) {
108
+ logger?.warn(`Could not compact the folder index at ${absolutePath}`, error);
109
+ }
110
+ }
111
+ }
112
+ return {
113
+ name: `file:${absolutePath}`,
114
+ get: (key) => map.get(key),
115
+ set: (key, id) => {
116
+ map.set(key, id);
117
+ mkdirSync(dirname(absolutePath), { recursive: true });
118
+ if (!existsSync(absolutePath)) {
119
+ writeFileSync(absolutePath, header);
120
+ }
121
+ appendFileSync(absolutePath, record(key, id));
122
+ },
123
+ knownContentHashes: (contentHashes) => {
124
+ const known = new Set([...map.keys()].map(contentHashFromFolderIndexKey));
125
+ return contentHashes.filter((contentHash) => known.has(contentHash));
126
+ },
127
+ entries: () => Object.fromEntries(map),
128
+ };
129
+ }
@@ -14,6 +14,7 @@
14
14
  * limitations under the License.
15
15
  */
16
16
  export * from './factory.js';
17
+ export * from './folderIndex.js';
17
18
  export * from './signer.js';
18
19
  export * from './upload.js';
19
20
  export * from '../types.js';
@@ -15,6 +15,8 @@
15
15
  */
16
16
  import { createReadStream, promises, statSync } from 'fs';
17
17
  import { lookup } from 'mime-types';
18
+ import { createHash } from 'node:crypto';
19
+ import { pipeline } from 'node:stream/promises';
18
20
  import { join } from 'path';
19
21
  import { Readable } from 'stream';
20
22
  import { TurboAuthenticatedBaseUploadService, defaultUploadServiceURL, } from '../common/upload.js';
@@ -84,4 +86,9 @@ export class TurboAuthenticatedUploadService extends TurboAuthenticatedBaseUploa
84
86
  createManifestStream(manifestBuffer) {
85
87
  return Readable.from(manifestBuffer);
86
88
  }
89
+ async computeContentHash(file) {
90
+ const hash = createHash('sha256');
91
+ await pipeline(createReadStream(file), hash);
92
+ return hash.digest('hex');
93
+ }
87
94
  }
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Copyright (C) 2022-2024 Permanent Data Solutions, Inc.
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ /**
17
+ * Internals shared by the folder index layers and by `uploadFolder`. Not part
18
+ * of the package's public surface.
19
+ */
20
+ /** Tag carrying the sha-256 of a file's own bytes, as lowercase hex. */
21
+ export const contentHashTagName = 'File-SHA256';
22
+ const contentHashRegex = /^[0-9a-f]{64}$/;
23
+ const folderIndexKeyRegex = /^[0-9a-f]{64}\.[0-9a-f]{64}$/;
24
+ /** True for a lowercase hex sha-256 digest of a file's bytes. */
25
+ export function isValidContentHash(hash) {
26
+ return hash !== undefined && contentHashRegex.test(hash);
27
+ }
28
+ /**
29
+ * A folder index key is `<sha-256 of the bytes>.<sha-256 of the tags>`.
30
+ *
31
+ * Both halves matter. Keying on the bytes alone would reuse an item whose tags
32
+ * are not the ones the caller asked for: an empty `a.css` and an empty `b.js`
33
+ * hash identically, and reusing one for the other would serve JavaScript as
34
+ * `text/css`. Keying on the tags as well means a reused data item is always
35
+ * exactly the data item this call would otherwise have created.
36
+ *
37
+ * The bytes half is kept in the clear so a layer that queries a gateway can
38
+ * recover it and filter on the `File-SHA256` tag.
39
+ */
40
+ export function isValidFolderIndexKey(key) {
41
+ return key !== undefined && folderIndexKeyRegex.test(key);
42
+ }
43
+ /** The bytes half of a folder index key. */
44
+ export function contentHashFromFolderIndexKey(key) {
45
+ return key.slice(0, 64);
46
+ }
47
+ async function sha256Hex(data) {
48
+ const digest = await crypto.subtle.digest('SHA-256', data);
49
+ return Buffer.from(digest).toString('hex');
50
+ }
51
+ /**
52
+ * Canonical encoding of a tag set: the same tags in a different order encode
53
+ * identically, because two data items that differ only in tag order are the
54
+ * same item as far as reuse goes.
55
+ *
56
+ * The sort compares code units rather than using `localeCompare`, which is not
57
+ * a total order over distinct strings and depends on the host's ICU data --
58
+ * NFC and NFD spellings of the same word compare equal without being equal, and
59
+ * two locales disagree on where an accented letter sorts. Either would let one
60
+ * tag set hash differently on two machines, which costs a reuse.
61
+ *
62
+ * The encoding has to be injective, not merely stable. An ANS-104 tag name and
63
+ * value are arbitrary byte arrays, so a delimiter-joined string collides: with
64
+ * a 0x00/0x01 join, a single tag whose value embeds both delimiters encodes
65
+ * exactly like two separate tags. A collision here is a wrong reuse, which is
66
+ * the one failure the composite key exists to make impossible. JSON escapes
67
+ * the delimiters, so no value can forge a record boundary.
68
+ */
69
+ function canonicalTags(tags) {
70
+ const pairs = tags
71
+ .map((tag) => [tag.name, tag.value])
72
+ .sort(([aName, aValue], [bName, bValue]) => {
73
+ const [a, b] = aName === bName ? [aValue, bValue] : [aName, bName];
74
+ return a < b ? -1 : a > b ? 1 : 0;
75
+ });
76
+ return new TextEncoder().encode(JSON.stringify(pairs));
77
+ }
78
+ /** The folder index key for a file with these bytes and these final tags. */
79
+ export async function folderIndexKey({ contentHash, tags, }) {
80
+ return `${contentHash}.${await sha256Hex(canonicalTags(tags))}`;
81
+ }
@@ -1 +1 @@
1
- {"version":3,"file":"arns.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/arns.ts"],"names":[],"mappings":"AAmBA,OAAO,EACL,UAAU,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,6BAA6B,EAC7B,YAAY,EACZ,eAAe,EACf,iBAAiB,EACjB,0BAA0B,EAG3B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAMtE,OAAO,EACL,wBAAwB,EACxB,sBAAsB,EACtB,oBAAoB,EACpB,gBAAgB,EAChB,mBAAmB,EACnB,yBAAyB,EACzB,2BAA2B,EAC3B,+BAA+B,EAC/B,uBAAuB,EACvB,4BAA4B,EAC5B,oBAAoB,EACpB,sBAAsB,EACtB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAyBrB,MAAM,MAAM,eAAe,GAAG;IAC5B,mBAAmB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC1E,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qBAAqB,CAAC,CAAC,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;CACzC,CAAC;AACF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,WAAW,CAAC,MAAM,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,eAAe,CAAC;QACvB,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,0BAA0B,CAAC,MAAM,EAAE;QACjC,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AACF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,eAAe,CAAC,MAAM,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,aAAa,CAAC,MAAM,EAAE;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,gBAAgB,CAAC,MAAM,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,iBAAiB,CAAC,MAAM,EAAE;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,oBAAoB,CAAC,MAAM,EAAE;QAC3B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,qBAAqB,CAAC,MAAM,EAAE;QAC5B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAClC,cAAc,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;KAClC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,wBAAwB,CAAC,MAAM,EAAE;QAC/B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,kBAAkB,CAAC,MAAM,EAAE;QACzB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AAUF,MAAM,MAAM,qBAAqB,GAAG;IAClC,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;CAC1E,CAAC;AAEF,wBAAgB,uBAAuB,CAAC,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAK1E;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,IAAI,EAAE,MAAM,GACX,MAAM,CASR;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAKvE;AAED,sFAAsF;AACtF,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,UAAU,CAiB1E;AAUD,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,GAC3B,MAAM,EAAE,GAAG,SAAS,CAEtB;AASD;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,gBAAgB,GACxB,eAAe,CAwCjB;AAyCD,wBAAsB,SAAS,CAC7B,OAAO,EAAE,gBAAgB,EACzB,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,wBAAwB,CACtB,MAAM,EAAE,2BAA2B,GAClC,OAAO,CAAC,6BAA6B,CAAC,CAAC;CAC3C,CAAC;AAEF,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA4Df;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAWf;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,yBAAyB,EAClC,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,EAC/B,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAqBf;AAED,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,uBAAuB,EAChC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,wBAAwB,EACjC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,2BAA2B,EACpC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,yBAAyB,EAClC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAwBf;AAED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,4BAA4B,EACrC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAmCf;AAED,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,+BAA+B,EACxC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,EAC/B,KAAK,CAAC,EAAE,qBAAqB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAaf"}
1
+ {"version":3,"file":"arns.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/arns.ts"],"names":[],"mappings":"AAmBA,OAAO,EACL,UAAU,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,6BAA6B,EAC7B,YAAY,EACZ,eAAe,EACf,iBAAiB,EACjB,0BAA0B,EAG3B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAMtE,OAAO,EACL,wBAAwB,EACxB,sBAAsB,EACtB,oBAAoB,EACpB,gBAAgB,EAChB,mBAAmB,EACnB,yBAAyB,EACzB,2BAA2B,EAC3B,+BAA+B,EAC/B,uBAAuB,EACvB,4BAA4B,EAC5B,oBAAoB,EACpB,sBAAsB,EACtB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAyBrB,MAAM,MAAM,eAAe,GAAG;IAC5B,mBAAmB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC1E,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,qBAAqB,CAAC,CAAC,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;CACzC,CAAC;AACF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,WAAW,CAAC,MAAM,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,eAAe,CAAC;QACvB,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,0BAA0B,CAAC,MAAM,EAAE;QACjC,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,eAAe,CAAC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AACF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,eAAe,CAAC,MAAM,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,aAAa,CAAC,MAAM,EAAE;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,gBAAgB,CAAC,MAAM,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,iBAAiB,CAAC,MAAM,EAAE;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,oBAAoB,CAAC,MAAM,EAAE;QAC3B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,qBAAqB,CAAC,MAAM,EAAE;QAC5B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAClC,cAAc,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;KAClC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,wBAAwB,CAAC,MAAM,EAAE;QAC/B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,kBAAkB,CAAC,MAAM,EAAE;QACzB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,eAAe,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClC,CAAC;AAUF,MAAM,MAAM,qBAAqB,GAAG;IAClC,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;CAC1E,CAAC;AAEF,wBAAgB,uBAAuB,CAAC,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAK1E;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,IAAI,EAAE,MAAM,GACX,MAAM,CASR;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAKvE;AAED,sFAAsF;AACtF,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,UAAU,CAiB1E;AAUD,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,GAC3B,MAAM,EAAE,GAAG,SAAS,CAEtB;AASD;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,gBAAgB,GACxB,eAAe,CAwCjB;AAyCD,wBAAsB,SAAS,CAC7B,OAAO,EAAE,gBAAgB,EACzB,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,wBAAwB,CACtB,MAAM,EAAE,2BAA2B,GAClC,OAAO,CAAC,6BAA6B,CAAC,CAAC;CAC3C,CAAC;AAEF,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA4Df;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAWf;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE,kBAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,yBAAyB,EAClC,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,EAC/B,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAqBf;AAED,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,uBAAuB,EAChC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,wBAAwB,EACjC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,2BAA2B,EACpC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,yBAAyB,EAClC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAwBf;AAED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,4BAA4B,EACrC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAmCf;AAED,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,+BAA+B,EACxC,KAAK,CAAC,EAAE,iBAAiB,GACxB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,EAC/B,KAAK,CAAC,EAAE,qBAAqB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAaf"}