@elaraai/e3-cloud-files 1.0.93 → 1.0.97

package/dist/src/index.d.ts

@@ -4,24 +4,27 @@
  */
 /**
  * `@elaraai/e3-cloud-files`: a solution's inputs that come from files in a
- * bucket.
+ * bucket, and its records seeded or fed from them.
  *
  * - `Files.source(name)`: a source, which each workspace binds to a bucket
  *   and a prefix; its `file`, `latest`, `folder` and `listing` declare the
- *   inputs its files fill, each an ordinary `e3.input`.
+ *   inputs its files fill, each an ordinary `e3.input`, a `file` or `latest`
+ *   typed when it names a `type`; its `mutation` declares a record's
+ *   mutation fed by its files.
  * - `Files.manifest(sources)`: the declarations `e3-cloud files deploy`
  *   deploys beside the package.
- * - `checkFiles`, `checkBindings`, `readPackageInputs`: the checks a deploy
- *   and a bind run.
+ * - `checkFiles`, `checkBindings`, `readPackageSurface`: the checks a deploy
+ *   and a bind run, against the package's inputs, records and mutations.
  *
- * A file arrives as it is, a `Blob`; the solution decodes it in East. The
- * sync is `@elaraai/e3-cloud-files/sync`; the East types are
+ * A file with no type arrives as it is, a `Blob`; the solution decodes it in
+ * East. A typed file arrives as the beast2 value it holds. The sync is
+ * `@elaraai/e3-cloud-files/sync`; the East types are
  * `@elaraai/e3-cloud-types`'s `Files*`.
  *
  * @packageDocumentation
  */
-export { Files, FilesSource, FilesDeclarationError, type FilesInput, type FilesSelection } from './authoring.js';
-export { checkFiles, checkBindings, readPackageInputs, shapeType, type PackageInput, type ObjectReader, type BindingPolicy, } from './check.js';
+export { Files, FilesSource, FilesDeclarationError, type FilesInput, type FilesMutation, type FilesSelection, type FilesKeySpec, type FilesTypedKeySpec, type FilesTypedSelection, } from './authoring.js';
+export { checkFiles, checkBindings, readPackageInputs, readPackageSurface, shapeType, type PackageInput, type PackageRecord, type PackageMutation, type PackageSurface, type ObjectReader, type BindingPolicy, } from './check.js';
 export { globMatcher, parseBucketUrl, keyProblem, prefixProblem, includeProblem, type BucketLocation } from './paths.js';
-export { FilesManifestType, FilesMappingType, FilesShapeType, FilesBindingType, FilesSyncReportType, FilesMappingReportType, FilesStatusType, FilesProblemType, FilesListingRowType, FilesFileType, FilesFolderType, FilesListingType, FILES_FILE_MAX_BYTES, FILES_FOLDER_ENTRY_MAX_BYTES, filesProblemText, type FilesManifest, type FilesMapping, type FilesShape, type FilesBinding, type FilesSyncReport, type FilesMappingReport, type FilesStatus, type FilesProblem, type FilesListingRow, } from '@elaraai/e3-cloud-types';
+export { FilesManifestType, FilesMappingType, FilesShapeType, FilesTargetType, FilesBindingType, FilesSyncReportType, FilesMappingReportType, FilesRefusalReasonType, FilesStatusType, FilesProblemType, FilesListingRowType, FilesFileType, FilesFolderType, FilesListingType, FILES_FILE_MAX_BYTES, FILES_FOLDER_ENTRY_MAX_BYTES, FILES_TYPED_VALUE_MAX_BYTES, FILES_SEGMENT_MAX_BYTES, filesProblemText, filesRefusalText, filesTargetText, parseFilesTarget, sameFilesTarget, type FilesManifest, type FilesMapping, type FilesShape, type FilesTarget, type FilesBinding, type FilesSyncReport, type FilesMappingReport, type FilesRefusalReason, type FilesStatus, type FilesProblem, type FilesListingRow, } from '@elaraai/e3-cloud-types';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACjH,OAAO,EACL,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,SAAS,EACT,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AACzH,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,mBAAmB,EACnB,sBAAsB,EACtB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,oBAAoB,EACpB,4BAA4B,EAC5B,gBAAgB,EAChB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,eAAe,GACrB,MAAM,yBAAyB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EACL,KAAK,EAAE,WAAW,EAAE,qBAAqB,EACzC,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,KAAK,mBAAmB,GAC9H,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,EACT,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AACzH,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,oBAAoB,EACpB,4BAA4B,EAC5B,2BAA2B,EAC3B,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,eAAe,GACrB,MAAM,yBAAyB,CAAC"}

package/dist/src/index.js

@@ -4,24 +4,27 @@
  */
 /**
  * `@elaraai/e3-cloud-files`: a solution's inputs that come from files in a
- * bucket.
+ * bucket, and its records seeded or fed from them.
  *
  * - `Files.source(name)`: a source, which each workspace binds to a bucket
  *   and a prefix; its `file`, `latest`, `folder` and `listing` declare the
- *   inputs its files fill, each an ordinary `e3.input`.
+ *   inputs its files fill, each an ordinary `e3.input`, a `file` or `latest`
+ *   typed when it names a `type`; its `mutation` declares a record's
+ *   mutation fed by its files.
  * - `Files.manifest(sources)`: the declarations `e3-cloud files deploy`
  *   deploys beside the package.
- * - `checkFiles`, `checkBindings`, `readPackageInputs`: the checks a deploy
- *   and a bind run.
+ * - `checkFiles`, `checkBindings`, `readPackageSurface`: the checks a deploy
+ *   and a bind run, against the package's inputs, records and mutations.
  *
- * A file arrives as it is, a `Blob`; the solution decodes it in East. The
- * sync is `@elaraai/e3-cloud-files/sync`; the East types are
+ * A file with no type arrives as it is, a `Blob`; the solution decodes it in
+ * East. A typed file arrives as the beast2 value it holds. The sync is
+ * `@elaraai/e3-cloud-files/sync`; the East types are
  * `@elaraai/e3-cloud-types`'s `Files*`.
  *
  * @packageDocumentation
  */
-export { Files, FilesSource, FilesDeclarationError } from './authoring.js';
-export { checkFiles, checkBindings, readPackageInputs, shapeType, } from './check.js';
+export { Files, FilesSource, FilesDeclarationError, } from './authoring.js';
+export { checkFiles, checkBindings, readPackageInputs, readPackageSurface, shapeType, } from './check.js';
 export { globMatcher, parseBucketUrl, keyProblem, prefixProblem, includeProblem } from './paths.js';
-export { FilesManifestType, FilesMappingType, FilesShapeType, FilesBindingType, FilesSyncReportType, FilesMappingReportType, FilesStatusType, FilesProblemType, FilesListingRowType, FilesFileType, FilesFolderType, FilesListingType, FILES_FILE_MAX_BYTES, FILES_FOLDER_ENTRY_MAX_BYTES, filesProblemText, } from '@elaraai/e3-cloud-types';
+export { FilesManifestType, FilesMappingType, FilesShapeType, FilesTargetType, FilesBindingType, FilesSyncReportType, FilesMappingReportType, FilesRefusalReasonType, FilesStatusType, FilesProblemType, FilesListingRowType, FilesFileType, FilesFolderType, FilesListingType, FILES_FILE_MAX_BYTES, FILES_FOLDER_ENTRY_MAX_BYTES, FILES_TYPED_VALUE_MAX_BYTES, FILES_SEGMENT_MAX_BYTES, filesProblemText, filesRefusalText, filesTargetText, parseFilesTarget, sameFilesTarget, } from '@elaraai/e3-cloud-types';
 //# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAwC,MAAM,gBAAgB,CAAC;AACjH,OAAO,EACL,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,SAAS,GAIV,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc,EAAuB,MAAM,YAAY,CAAC;AACzH,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,mBAAmB,EACnB,sBAAsB,EACtB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,oBAAoB,EACpB,4BAA4B,EAC5B,gBAAgB,GAUjB,MAAM,yBAAyB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EACL,KAAK,EAAE,WAAW,EAAE,qBAAqB,GAE1C,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,GAOV,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc,EAAuB,MAAM,YAAY,CAAC;AACzH,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,oBAAoB,EACpB,4BAA4B,EAC5B,2BAA2B,EAC3B,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,eAAe,GAYhB,MAAM,yBAAyB,CAAC"}

package/dist/src/s3/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/s3/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAiF,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAClI,OAAO,EAA0C,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE7F,sFAAsF;AACtF,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+BAA+B;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAgCD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,GAAG,WAAW,CAgEjF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/s3/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAiF,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAClI,OAAO,EAA0C,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE7F,sFAAsF;AACtF,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+BAA+B;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAgCD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,GAAG,WAAW,CAqFjF"}

package/dist/src/s3/index.js

@@ -63,6 +63,22 @@ function unreadable(err) {
  */
 export function s3FilesReader(client, location) {
     const absolute = (key) => `${location.prefix}${key}`;
+    /** A file, or a range of it, at the version its listing gave; a refusal of the file's own is unreadable. */
+    const get = async (file, range) => {
+        let object;
+        try {
+            object = await client.send(new GetObjectCommand({ Bucket: location.bucket, Key: absolute(file.key), IfMatch: `"${file.etag}"`, ...(range !== undefined && { Range: range }) }));
+        }
+        catch (err) {
+            const reason = unreadable(err);
+            if (reason !== null)
+                throw new FilesUnreadableError(file.key, reason);
+            throw err;
+        }
+        if (object.Body === undefined)
+            throw new Error(`s3://${location.bucket}/${absolute(file.key)} answered no bytes`);
+        return object;
+    };
     return {
         async *list(prefix, options = {}) {
             let token;
@@ -114,18 +130,7 @@ export function s3FilesReader(client, location) {
             return { key, size: object.Size ?? 0, modified: object.LastModified ?? new Date(0), etag: bare(object.ETag) };
         },
         async read(file) {
-            let object;
-            try {
-                object = await client.send(new GetObjectCommand({ Bucket: location.bucket, Key: absolute(file.key), IfMatch: `"${file.etag}"` }));
-            }
-            catch (err) {
-                const reason = unreadable(err);
-                if (reason !== null)
-                    throw new FilesUnreadableError(file.key, reason);
-                throw err;
-            }
-            if (object.Body === undefined)
-                throw new Error(`s3://${location.bucket}/${absolute(file.key)} answered no bytes`);
+            const object = await get(file);
             if (object.ContentLength !== undefined && object.ContentLength !== file.size) {
                 // Never buffered: its size was never checked
                 object.Body.destroy?.();
@@ -133,6 +138,23 @@ export function s3FilesReader(client, location) {
             }
             return object.Body.transformToByteArray();
         },
+        async readRange(file, start, end) {
+            if (end <= start)
+                return new Uint8Array(0);
+            // A range's end is inclusive to the bucket
+            const object = await get(file, `bytes=${start}-${end - 1}`);
+            return object.Body.transformToByteArray();
+        },
+        async open(file, options = {}) {
+            const start = options.start ?? 0;
+            const object = await get(file, start > 0 ? `bytes=${start}-` : undefined);
+            if (object.ContentLength !== undefined && object.ContentLength !== file.size - start) {
+                object.Body.destroy?.();
+                throw new FilesUnreadableError(file.key, `changed since it was listed (${object.ContentLength + start} bytes, not the ${file.size} listed): the next sync takes it`);
+            }
+            // A bucket's body is a stream of chunks
+            return object.Body;
+        },
     };
 }
 //# sourceMappingURL=index.js.map

package/dist/src/s3/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/s3/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,kBAAkB,EAAiB,MAAM,oBAAoB,CAAC;AAClI,OAAO,EAAE,oBAAoB,EAAsC,MAAM,mBAAmB,CAAC;AAS7F,yDAAyD;AACzD,SAAS,IAAI,CAAC,IAAwB;IACpC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,6DAA6D;AAC7D,SAAS,OAAO,CAAC,GAAY;IAC3B,OAAO,GAAG,YAAY,kBAAkB,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC;AAC5I,CAAC;AAED,+DAA+D;AAC/D,SAAS,MAAM,CAAC,GAAY;IAC1B,OAAO,GAAG,YAAY,kBAAkB,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC;AACpH,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,GAAY;IAC9B,IAAI,CAAC,CAAC,GAAG,YAAY,kBAAkB,CAAC;QAAE,OAAO,IAAI,CAAC;IACtD,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,oBAAoB,CAAC,CAAC,OAAO,qDAAqD,CAAC;QACxF,KAAK,oBAAoB,CAAC,CAAC,OAAO,uEAAuE,CAAC;QAC1G,KAAK,cAAc,CAAC,CAAC,OAAO,kFAAkF,CAAC;QAC/G,KAAK,WAAW,CAAC,CAAC,OAAO,6BAA6B,CAAC;QACvD,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9F,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,QAAoB;IAClE,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;IAC7D,OAAO;QACL,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;YAC9B,IAAI,KAAyB,CAAC;YAC9B,GAAG,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;oBACtD,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC;oBACxB,iBAAiB,EAAE,KAAK;oBACxB,4EAA4E;oBAC5E,GAAG,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;iBAC1D,CAAC,CAAC,CAAC;gBACJ,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oBACzC,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS;wBAAE,SAAS;oBACvC,MAAM;wBACJ,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;wBAC7C,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC;wBACtB,QAAQ,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC;wBAC5C,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;qBACxB,CAAC;gBACJ,CAAC;gBACD,KAAK,GAAG,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,CAAC,QAAQ,KAAK,KAAK,SAAS,EAAE;QAChC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG;YACZ,qEAAqE;YACrE,kDAAkD;YAClD,IAAI,IAAI,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACrH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;oBAAE,MAAM,GAAG,CAAC;gBAC5B,mFAAmF;gBACnF,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvG,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnH,CAAC;gBAAC,OAAO,OAAO,EAAE,CAAC;oBACjB,IAAI,OAAO,CAAC,OAAO,CAAC;wBAAE,OAAO,IAAI,CAAC;oBAClC,MAAM,OAAO,CAAC;gBAChB,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,MAAM,EAAE,GAAG,KAAK,QAAQ,CAAC,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC/C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI;YACb,IAAI,MAAM,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;YACpI,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,MAAM,KAAK,IAAI;oBAAE,MAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBACtE,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClH,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7E,6CAA6C;gBAC5C,MAAM,CAAC,IAAiC,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtD,MAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,gCAAgC,MAAM,CAAC,aAAa,mBAAmB,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;YAC/J,CAAC;YACD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/s3/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,kBAAkB,EAAiB,MAAM,oBAAoB,CAAC;AAClI,OAAO,EAAE,oBAAoB,EAAsC,MAAM,mBAAmB,CAAC;AAS7F,yDAAyD;AACzD,SAAS,IAAI,CAAC,IAAwB;IACpC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,6DAA6D;AAC7D,SAAS,OAAO,CAAC,GAAY;IAC3B,OAAO,GAAG,YAAY,kBAAkB,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC;AAC5I,CAAC;AAED,+DAA+D;AAC/D,SAAS,MAAM,CAAC,GAAY;IAC1B,OAAO,GAAG,YAAY,kBAAkB,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC;AACpH,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,GAAY;IAC9B,IAAI,CAAC,CAAC,GAAG,YAAY,kBAAkB,CAAC;QAAE,OAAO,IAAI,CAAC;IACtD,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,oBAAoB,CAAC,CAAC,OAAO,qDAAqD,CAAC;QACxF,KAAK,oBAAoB,CAAC,CAAC,OAAO,uEAAuE,CAAC;QAC1G,KAAK,cAAc,CAAC,CAAC,OAAO,kFAAkF,CAAC;QAC/G,KAAK,WAAW,CAAC,CAAC,OAAO,6BAA6B,CAAC;QACvD,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9F,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,QAAoB;IAClE,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;IAC7D,4GAA4G;IAC5G,MAAM,GAAG,GAAG,KAAK,EAAE,IAAiB,EAAE,KAAc,EAAE,EAAE;QACtD,IAAI,MAAM,CAAC;QACX,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAClL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACtE,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClH,OAAO,MAAmE,CAAC;IAC7E,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;YAC9B,IAAI,KAAyB,CAAC;YAC9B,GAAG,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;oBACtD,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC;oBACxB,iBAAiB,EAAE,KAAK;oBACxB,4EAA4E;oBAC5E,GAAG,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;iBAC1D,CAAC,CAAC,CAAC;gBACJ,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oBACzC,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS;wBAAE,SAAS;oBACvC,MAAM;wBACJ,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;wBAC7C,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC;wBACtB,QAAQ,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC;wBAC5C,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;qBACxB,CAAC;gBACJ,CAAC;gBACD,KAAK,GAAG,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,CAAC,QAAQ,KAAK,KAAK,SAAS,EAAE;QAChC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG;YACZ,qEAAqE;YACrE,kDAAkD;YAClD,IAAI,IAAI,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACrH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;oBAAE,MAAM,GAAG,CAAC;gBAC5B,mFAAmF;gBACnF,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvG,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnH,CAAC;gBAAC,OAAO,OAAO,EAAE,CAAC;oBACjB,IAAI,OAAO,CAAC,OAAO,CAAC;wBAAE,OAAO,IAAI,CAAC;oBAClC,MAAM,OAAO,CAAC;gBAChB,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,MAAM,EAAE,GAAG,KAAK,QAAQ,CAAC,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC/C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI;YACb,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7E,6CAA6C;gBAC5C,MAAM,CAAC,IAAiC,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtD,MAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,gCAAgC,MAAM,CAAC,aAAa,mBAAmB,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;YAC/J,CAAC;YACD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG;YAC9B,IAAI,GAAG,IAAI,KAAK;gBAAE,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3C,2CAA2C;YAC3C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5D,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,EAAE;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;gBACpF,MAAM,CAAC,IAAiC,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtD,MAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,gCAAgC,MAAM,CAAC,aAAa,GAAG,KAAK,mBAAmB,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;YACvK,CAAC;YACD,wCAAwC;YACxC,OAAO,MAAM,CAAC,IAA4C,CAAC;QAC7D,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/src/storage/index.d.ts

@@ -2,15 +2,84 @@
  * Copyright (c) 2025 Elara AI Pty Ltd
  * Licensed under BSL 1.1. See LICENSE for details.
  */
-import { type StorageBackend } from '@elaraai/e3-core';
-import type { FilesWriter } from '../sync/engine.js';
+import { type StorageBackend, type TaskRunner } from '@elaraai/e3-core';
+import { type FilesRefusalReason } from '@elaraai/e3-cloud-types';
+import { type FilesWriter } from '../sync/engine.js';
+/** What the writer is given beside the store. */
+export interface StorageWriterOptions {
+    /** The runner that takes a typed collection in by pieces, and runs a fed mutation; without one, a typed file that is a collection the store does not know, or a feed, is refused */
+    readonly runner?: TaskRunner;
+    /** Where a typed file's bytes are streamed to before e3 takes them in: a directory of the caller's (default the system's temporary directory) */
+    readonly scratchDir?: string;
+    /** Who a feed's commit records as its actor (default `files`) */
+    readonly actor?: string;
+    /** How long a fed mutation may run, in ms (default e3's) */
+    readonly feedBudgetMs?: number;
+}
+/**
+ * A refusal e3's door answered, as the sync reports it; null for an error the
+ * door did not refuse.
+ *
+ * @param err - What the door threw
+ * @param type - The mapping's type
+ * @returns The refusal, or null
+ */
+export declare function doorRefusal(err: unknown): FilesRefusalReason | null;
 /**
  * The sync's writer over an e3 store, for one workspace.
  *
  * @param storage - The store: a local repository's (`LocalStorage`) or the platform's
  * @param repo - The repository: a local repository's path, or the platform's repository name
  * @param ws - The workspace
+ * @param options - The runner, the scratch directory and the actor
+ * @returns The writer
+ */
+export declare function storageWriter(storage: StorageBackend, repo: string, ws: string, options?: StorageWriterOptions): FilesWriter;
+/**
+ * Stages a typed file's bytes in the object store as the object their
+ * SHA-256 names, hashing them as they stream: the platform's typed intake,
+ * never on a disk. A stage the signal stops keeps nothing: the next round
+ * starts the file again.
+ */
+export interface FilesStaging {
+    /**
+     * @param repo - The repository, as the storage names it
+     * @param open - The file's bytes, from a byte on: opened at 0, or where a stage left off
+     * @param size - The file's size, as its listing gave it
+     * @param options - The round's signal, and where an earlier stage of the same file stood (its `progress`)
+     * @returns The bytes' SHA-256, the object placed; or `aborted` when the
+     *   signal stopped it, with where it stood for the next round to resume
+     *   (null when nothing was kept)
+     */
+    stage(repo: string, open: (start: number) => Promise<AsyncIterable<Uint8Array>>, size: number, options: {
+        readonly signal?: AbortSignal;
+        readonly resume?: string;
+    }): Promise<{
+        sha256: string;
+    } | {
+        aborted: true;
+        progress: string | null;
+    }>;
+}
+/** What the cloud's writer is given beside the store. */
+export interface CloudWriterOptions extends Omit<StorageWriterOptions, 'scratchDir'> {
+    /** The round's runner: takes a typed collection in by pieces, runs a fed mutation */
+    readonly runner: TaskRunner;
+    /** Where a typed file's bytes are staged: the platform's object store */
+    readonly staging: FilesStaging;
+}
+/**
+ * The sync's writer for a deployed workspace: the store's writer, with a
+ * typed file taken in through the platform's staging and e3's door for a
+ * delivery the store holds (`datasetAdoptObject`), so no byte of it passes
+ * through a disk (D7), and an intake the round's deadline stops resumes in
+ * the next round with the pieces it finished remembered.
+ *
+ * @param storage - The platform's storage
+ * @param repo - The repository, as the storage names it
+ * @param ws - The workspace
+ * @param options - The runner, the staging, the actor
  * @returns The writer
  */
-export declare function storageWriter(storage: StorageBackend, repo: string, ws: string): FilesWriter;
+export declare function cloudWriter(storage: StorageBackend, repo: string, ws: string, options: CloudWriterOptions): FilesWriter;
 //# sourceMappingURL=index.d.ts.map

package/dist/src/storage/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAOL,KAAK,cAAc,EACpB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,WAAW,EAAiB,MAAM,mBAAmB,CAAC;AAgDpE;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,WAAW,CA4B5F"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA+CH,OAAO,EAcL,KAAK,cAAc,EACnB,KAAK,UAAU,EAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAA0E,KAAK,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC1I,OAAO,EAEkE,KAAK,WAAW,EAExF,MAAM,mBAAmB,CAAC;AAwD3B,iDAAiD;AACjD,MAAM,WAAW,oBAAoB;IACnC,oLAAoL;IACpL,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,iJAAiJ;IACjJ,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,4DAA4D;IAC5D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CASnE;AA2BD;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,WAAW,CA2GhI;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;;;OAQG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;QAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,OAAO,EAAE,IAAI,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CAChP;AAED,yDAAyD;AACzD,MAAM,WAAW,kBAAmB,SAAQ,IAAI,CAAC,oBAAoB,EAAE,YAAY,CAAC;IAClF,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;CAChC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,WAAW,CAqDvH"}

package/dist/src/storage/index.js

@@ -4,30 +4,61 @@
  */
 /**
  * `@elaraai/e3-cloud-files/storage`: the sync's writer over an e3 store, the
- * one a local repository and a deployed workspace both write through.
+ * one a local repository writes through, and the base of a deployed
+ * workspace's.
  *
- * A `file`, `latest` or `listing` input is written as any input is
- * (`workspaceSetDataset`). A `folder` is updated a piece at a time: e3 stores
- * a collection as pieces, each holding the entries between two keys, so only
- * the pieces holding a changed or removed file are read and cut again, and
- * every other piece is carried over as it is stored (`storeCollection`), never
- * read. However large the folder, a sync holds the pieces it changes and the
- * files it read. The new folder is then set as the input's value by e3's door
- * for a value the store holds (`datasetSetStored`), which checks its type,
- * holds the dataset's write lock and versions it as any input write.
+ * A `file`, `latest` or `listing` input with no type is written as any input
+ * is (`workspaceSetDataset`). A `folder` is updated a piece at a time: e3
+ * stores a collection as pieces, each holding the entries between two keys,
+ * so only the pieces holding a changed or removed file are read and cut
+ * again, and every other piece is carried over as it is stored
+ * (`storeCollection`), never read. However large the folder, a sync holds
+ * the pieces it changes and the files it read. The new folder is then set as
+ * the input's value by e3's door for a value the store holds
+ * (`datasetSetStored`), which checks its type, holds the dataset's write lock
+ * and versions it as any input write.
+ *
+ * A typed file is taken in through e3's door for a file (`datasetAdoptFile`):
+ * its bytes are streamed to a file of the repository's own scratch, hashed as
+ * they go, and the door takes the file in as the value it holds, a collection
+ * by intake units on the runner given, remembered by the bytes' hash so the
+ * same bytes taken again cost their hash and nothing else. (A local
+ * repository may use its disk; the platform's writer stages in its bucket
+ * instead, and never on a disk.) A fed mutation runs through e3's own
+ * `recordMutate`, the file's hash its idempotency key, so a file applied
+ * once is never applied twice.
+ *
+ * {@link cloudWriter} is the platform's: the same, but a typed file staged
+ * in the object store through {@link FilesStaging} (never a disk) and taken
+ * in through e3's door for a delivery the store holds, resumable across the
+ * platform's rounds.
  *
  * Needs `@elaraai/e3-core`, which a solution's own code never imports: the
  * CLI and the platform do.
  *
  * @packageDocumentation
  */
-import { StringType, compareFor, decodeBeast2For, variant } from '@elaraai/east';
-import { DatasetSegments, datasetSetStored, storeCollection, workspaceGetDatasetHash, workspaceSetDataset, } from '@elaraai/e3-core';
+import { createHash } from 'node:crypto';
+import { createWriteStream } from 'node:fs';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { StringType, compareFor, decodeBeast2For, none, some, variant } from '@elaraai/east';
+import { DatasetSegments, DatasetTypeMismatchError, DeliveryRefusedError, ObjectNotFoundError, datasetAdoptFile, datasetAdoptObject, datasetSetStored, recordHistory, recordMutate, storeCollection, workspaceGetDatasetHash, workspaceSetDataset, } from '@elaraai/e3-core';
 import { FilesFileType, FilesFolderType, FilesListingType } from '@elaraai/e3-cloud-types';
+import { FilesUnreadableError, } from '../sync/engine.js';
 /** An input's path: `.inputs.<input>`. */
 function inputPath(input) {
     return [variant('field', 'inputs'), variant('field', input)];
 }
+/** A record's path: `.records.<record>`. */
+function recordPath(record) {
+    return [variant('field', 'records'), variant('field', record)];
+}
+/** The most commits a seed walks back for a record's last seed. */
+const SEED_WALK_LIMIT = 100;
 const compareKeys = compareFor(StringType);
 const decodeFolder = decodeBeast2For(FilesFolderType);
 /** Entries in a Dict's order. */
@@ -73,15 +104,61 @@ async function folderSources(storage, repo, hash, changes) {
         sources.push({ stored: pieces.hash, from: next, to: pieces.counts.length });
     return sources;
 }
+/**
+ * A refusal e3's door answered, as the sync reports it; null for an error the
+ * door did not refuse.
+ *
+ * @param err - What the door threw
+ * @param type - The mapping's type
+ * @returns The refusal, or null
+ */
+export function doorRefusal(err) {
+    if (err instanceof DatasetTypeMismatchError)
+        return variant('type_mismatch', { expected: err.mismatch.declared, actual: err.mismatch.given });
+    if (err instanceof DeliveryRefusedError) {
+        // east's intake: "segment N of the delivery, at offset X, holds B bytes, more than the M a segment is read in — write it again …"
+        const segment = /holds ([0-9]+) bytes, more than the ([0-9]+) a segment/i.exec(err.refusal);
+        if (segment !== null)
+            return variant('segment_too_large', { bytes: BigInt(Number(segment[1])), limit: BigInt(Number(segment[2])) });
+        return variant('other', `the runner refused it: ${err.refusal}`);
+    }
+    return null;
+}
+/**
+ * Streams a file's bytes to a scratch file, hashing them as they go.
+ *
+ * @returns The scratch file's path and the bytes' SHA-256
+ */
+async function spool(source, file, dir, signal) {
+    await mkdir(dir, { recursive: true });
+    const path = join(dir, `files-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.beast2`);
+    const hash = createHash('sha256');
+    const stream = Readable.from(await source.open(file));
+    const hashing = async function* (chunks) {
+        for await (const chunk of chunks) {
+            hash.update(chunk);
+            yield chunk;
+        }
+    };
+    try {
+        await pipeline(stream, hashing, createWriteStream(path), signal === undefined ? {} : { signal });
+    }
+    catch (err) {
+        await rm(path, { force: true });
+        throw err;
+    }
+    return { path, sha256: hash.digest('hex') };
+}
 /**
  * The sync's writer over an e3 store, for one workspace.
  *
  * @param storage - The store: a local repository's (`LocalStorage`) or the platform's
  * @param repo - The repository: a local repository's path, or the platform's repository name
  * @param ws - The workspace
+ * @param options - The runner, the scratch directory and the actor
  * @returns The writer
  */
-export function storageWriter(storage, repo, ws) {
+export function storageWriter(storage, repo, ws, options = {}) {
     const current = async (input) => (await workspaceGetDatasetHash(storage, repo, ws, inputPath(input))).hash;
     const written = async (input) => {
         const hash = await current(input);
@@ -89,6 +166,7 @@ export function storageWriter(storage, repo, ws) {
             throw new Error(`input ${input} is unassigned after it was written`);
         return hash;
     };
+    const scratch = options.scratchDir ?? tmpdir();
     return {
         current,
         async setFile(input, bytes) {
@@ -108,6 +186,173 @@ export function storageWriter(storage, repo, ws) {
             // sets it as the input's value, checked and versioned, reading nothing
             return (await datasetSetStored(storage, repo, ws, inputPath(input), stored)).hash;
         },
+        async takeTyped(input, file, type, source, intake) {
+            let spooled;
+            try {
+                spooled = await spool(source, file, scratch, intake.signal);
+            }
+            catch (err) {
+                if (err instanceof FilesUnreadableError)
+                    return { status: 'refused', reason: variant('unreadable', err.reason) };
+                throw err;
+            }
+            try {
+                const adopted = await datasetAdoptFile(storage, repo, ws, inputPath(input), spooled.path, {
+                    expectHash: spooled.sha256,
+                    ...(options.runner !== undefined && { runner: options.runner }),
+                    ...(intake.signal !== undefined && { signal: intake.signal }),
+                });
+                return {
+                    status: 'taken',
+                    hash: adopted.hash,
+                    sha256: spooled.sha256,
+                    taken: { key: file.key, type, rows: adopted.rows === null ? none : some(BigInt(adopted.rows)), how: variant(adopted.segments === null ? 'carried' : 'split', null) },
+                };
+            }
+            catch (err) {
+                const reason = doorRefusal(err);
+                if (reason !== null)
+                    return { status: 'refused', reason };
+                throw err;
+            }
+            finally {
+                await rm(spooled.path, { force: true });
+            }
+        },
+        async recordState(record) {
+            const { hash } = await workspaceGetDatasetHash(storage, repo, ws, recordPath(record));
+            if (hash === null)
+                return { pending: true };
+            // The head, and the commits since the last seed, walked back a bounded way
+            const history = await recordHistory(storage, repo, ws, record, { limit: SEED_WALK_LIMIT });
+            const head = history[0];
+            if (head === undefined)
+                return { pending: true };
+            const seed = history.findIndex((entry) => entry.commit.mutation === '$init' || entry.commit.mutation === '$seed');
+            return { pending: false, head: head.hash, sinceSeed: seed < 0 ? null : seed };
+        },
+        async feed(record, mutation, file, type, source, feed) {
+            if (options.runner === undefined)
+                throw new Error('a fed mutation needs a runner');
+            // A record with no state yet: nothing is read for it
+            const { hash } = await workspaceGetDatasetHash(storage, repo, ws, recordPath(record));
+            if (hash === null)
+                return { status: 'waiting', reason: 'the record has no state yet' };
+            let bytes;
+            try {
+                const chunks = [];
+                for await (const chunk of await source.open(file))
+                    chunks.push(chunk);
+                bytes = Buffer.concat(chunks);
+            }
+            catch (err) {
+                if (err instanceof FilesUnreadableError)
+                    return { status: 'refused', reason: variant('unreadable', err.reason), sha256: '' };
+                throw err;
+            }
+            const sha256 = createHash('sha256').update(bytes).digest('hex');
+            // Bytes applied before, under any name, are applied no more
+            if (feed.applied?.has(sha256) === true)
+                return { status: 'applied', sha256 };
+            const outcome = await recordMutate(storage, options.runner, repo, ws, record, mutation, [bytes], {
+                actor: options.actor ?? 'files',
+                idempotencyKey: sha256,
+                ...(options.feedBudgetMs !== undefined && { budgetMs: options.feedBudgetMs }),
+                ...(feed.signal !== undefined && { signal: feed.signal }),
+            });
+            void type;
+            switch (outcome.kind) {
+                case 'committed':
+                    return { status: 'committed', commit: outcome.commitHash, sha256 };
+                case 'conflict':
+                    // The record was busy: the next sync tries the file again
+                    return { status: 'waiting', reason: `the record was busy (${outcome.detail ?? `the write lost the race ${outcome.attempts} times`}): tried again by the next sync` };
+                case 'invalid':
+                    return { status: 'refused', reason: variant('mutation', { outcome: 'invalid', message: outcome.message }), sha256 };
+                case 'failed':
+                    // Cut short by the round's end: not the file's fault
+                    if (feed.signal?.aborted === true)
+                        return { status: 'waiting', reason: 'the round ended before the mutation finished: tried again by the next sync' };
+                    return { status: 'refused', reason: variant('mutation', { outcome: 'failed', message: outcome.stderr.trim().split('\n').slice(-3).join(' ') }), sha256 };
+                case 'timed_out':
+                    return { status: 'refused', reason: variant('mutation', { outcome: 'timed_out', message: `after ${outcome.ms} ms` }), sha256 };
+            }
+        },
+    };
+}
+/**
+ * The sync's writer for a deployed workspace: the store's writer, with a
+ * typed file taken in through the platform's staging and e3's door for a
+ * delivery the store holds (`datasetAdoptObject`), so no byte of it passes
+ * through a disk (D7), and an intake the round's deadline stops resumes in
+ * the next round with the pieces it finished remembered.
+ *
+ * @param storage - The platform's storage
+ * @param repo - The repository, as the storage names it
+ * @param ws - The workspace
+ * @param options - The runner, the staging, the actor
+ * @returns The writer
+ */
+export function cloudWriter(storage, repo, ws, options) {
+    const base = storageWriter(storage, repo, ws, { runner: options.runner, ...(options.actor !== undefined && { actor: options.actor }), ...(options.feedBudgetMs !== undefined && { feedBudgetMs: options.feedBudgetMs }) });
+    return {
+        ...base,
+        async takeTyped(input, file, type, source, intake) {
+            // The engine resumes only the intake of this same file, by its key and ETag
+            const progress = (sha256, staging) => ({ key: file.key, etag: file.etag, sha256, staging });
+            let sha256 = intake.resume?.sha256 ?? null;
+            const stage = async (resume) => {
+                let staged;
+                try {
+                    staged = await options.staging.stage(repo, (start) => source.open(file, { start }), file.size, {
+                        ...(intake.signal !== undefined && { signal: intake.signal }),
+                        ...(resume !== null && { resume }),
+                    });
+                }
+                catch (err) {
+                    if (err instanceof FilesUnreadableError)
+                        return { status: 'refused', reason: variant('unreadable', err.reason) };
+                    throw err;
+                }
+                if ('aborted' in staged)
+                    return { status: 'yielded', taking: staged.progress === null ? null : progress(null, staged.progress) };
+                return staged.sha256;
+            };
+            if (sha256 === null) {
+                const staged = await stage(intake.resume?.staging ?? null);
+                if (typeof staged !== 'string')
+                    return staged;
+                sha256 = staged;
+            }
+            for (;;) {
+                try {
+                    const adopted = await datasetAdoptObject(storage, repo, ws, inputPath(input), sha256, options.runner, intake.signal === undefined ? {} : { signal: intake.signal });
+                    return {
+                        status: 'taken',
+                        hash: adopted.hash,
+                        sha256,
+                        taken: { key: file.key, type, rows: adopted.rows === null ? none : some(BigInt(adopted.rows)), how: variant(adopted.segments === null ? 'carried' : 'split', null) },
+                    };
+                }
+                catch (err) {
+                    // Stopped at the round's deadline, the pieces it finished remembered: the next round resumes it
+                    if (intake.signal?.aborted === true)
+                        return { status: 'yielded', taking: progress(sha256, null) };
+                    // The delivery a resumed intake names is gone (collected since): staged again
+                    if (err instanceof ObjectNotFoundError && intake.resume !== undefined) {
+                        const staged = await stage(null);
+                        if (typeof staged !== 'string')
+                            return staged;
+                        sha256 = staged;
+                        continue;
+                    }
+                    const reason = doorRefusal(err);
+                    if (reason !== null)
+                        return { status: 'refused', reason };
+                    throw err;
+                }
+            }
+        },
     };
 }
 //# sourceMappingURL=index.js.map