gina 0.6.7 → 0.6.8

package/framework/{v0.6.7 → v0.6.8}/lib/storage/src/local.js

@@ -72,22 +72,36 @@ var TMP_DIR = '.tmp';
  * seam in the tree (job-store, session-store, the upload mover) rather than
  * introducing a promise style the surrounding code does not use.
  *
- * Verbs arrive WITH the strategy that needs them: `getRange` (Range serving),
- * the resumable trio (`createUpload` / `writeSegment` / `finalize`) and
- * `findByDigest` (content-addressed dedup) are deliberately absent from v0 and
- * will be capability-gated when they land.
+ * Verbs arrive WITH the strategy that needs them. `getRange` (Range serving)
+ * has LANDED on every local strategy — it is contract, not strategy detail, and
+ * a Range verb that existed only on `stream` would serve nobody, since `sharded`
+ * is the zero-config default. The five resumable verbs (`createUpload` /
+ * `writeSegment` / `statUpload` / `finalize` / `abortUpload`) are `stream`-only
+ * and `findByDigest` (content-addressed dedup) `cas`-only; both stay
+ * capability-gated (`capabilities.resumable`, `capabilities.dedup`).
+ *
+ * `capabilities.ranges` is consumed by the framework itself: the controller's
+ * `serveFromStorage()` facade answers 206/416/304 with `Accept-Ranges`/
+ * `Content-Range` on both engines, gated on this flag — an adapter that
+ * cannot serve ranges is transparently answered with the full 200.
  *
  * @typedef  {Object} StorageDriver
  * @property {function(object, object, function): void} put          - Store a readable stream; `fn(err, {@link StoragePutResult})`.
- * @property {function(string, function): void}         get          - Open a readable stream for `key`; `fn(err, stream)`. Errors on an unknown key — see `stat` for the existence question.
+ * @property {function(string, function): void}         get          - Open a readable stream for `key`; `fn(err, stream)`. Errors on an unknown key — see `stat` for the existence question. Read-verb errors carry a machine `code` (`STORAGE_NO_OBJECT` / `STORAGE_RANGE_UNSATISFIABLE` / `STORAGE_INVALID_RANGE` — see `util.codedError`) so an HTTP serving layer maps 404/416/400 without parsing message text.
+ * @property {function(string, number, number, function): void} getRange - Open a readable stream over an INCLUSIVE byte range; `fn(err, stream)`. An over-long `end` is clamped; an unsatisfiable `start` errors (the caller's 416).
  * @property {function(string, function): void}         stat         - Metadata for `key`; `fn(err, meta|null)`. `null` (not an error) when unknown.
  * @property {function(string, function=): void}        release      - Delete object and metadata; `fn(err, existed)`.
- * @property {function(string, function): void}         resolve      - How to serve `key`; `fn(err, {kind:'path', path})`.
+ * @property {function(string, ?object, function): void} resolve     - How to serve `key` — `resolve(key[, opts], fn)`; `fn(err, {kind:'path', path} | {kind:'inline'} | {kind:'url', url, expiresAt})`. `opts` carries offload response overrides (`contentType`/`download`/`filename`/`cacheControl` → the provider's signed `response-*` params); local strategies accept and IGNORE it, so one facade call site serves every driver.
  * @property {function(): void}                         close        - Release the metadata handle. Teardown/tests only.
  * @property {object}                                   capabilities - What this driver can do; every consumer must branch on it rather than assume.
  * @property {function(function): void}                 [stats]      - Driver statistics for the operator surface (`storage:stats`); `fn(err, {name, strategy, root, capabilities, store})` — `store` is the metadata store's aggregate counts, or `null` when the store reports no stats. Present on both local strategies.
- * @property {function((object|function)=, function=): void} [sweepNow] - cas-only: run one GC pass now; `{dryRun: true}` lists collectable keys without touching anything. `fn(err, {collected, drained}|{collectable, drained})`. A strategy without a sweep simply lacks the verb — callers branch, never assume.
+ * @property {function((object|function)=, function=): void} [sweepNow] - cas-only: run one GC pass now; `{dryRun: true}` lists collectable keys without touching anything. `fn(err, {collected, drained}|{collectable, drained})`. A strategy without a sweep simply lacks the verb — callers branch, never assume. NB `stream` reclaims abandoned upload sessions on its own schedule but exposes no operator door in v1, so it correctly lacks this verb.
  * @property {function(object, function): void}         [verify]     - cas-only: files↔rows consistency scan (age-gated); see `src/local-cas`.
+ * @property {function(object, function): void}         [createUpload] - stream-only: open a resumable upload session; `fn(err, {uploadId, chunkSize, expectedSize})`. `expectedSize` is REQUIRED — see `src/local-stream`.
+ * @property {function(string, number, object, function): void} [writeSegment] - stream-only: write one segment at a byte offset, out of order and idempotently; `fn(err, {offset, length, received})`.
+ * @property {function(string, function): void}         [statUpload]   - stream-only: what landed and what is missing; `fn(err, {expectedSize, received, missing, complete, …})`. The resumable twin of `stat()`.
+ * @property {function(string, function): void}         [finalize]     - stream-only: verify coverage as an interval union, publish, index; `fn(err, {key, size, contentType})`. Idempotent, with a heal path.
+ * @property {function(string, function=): void}        [abortUpload]  - stream-only: discard a session; `fn(err, {aborted})`. Idempotent.
  */
 
 /**
@@ -103,6 +117,12 @@ var TMP_DIR = '.tmp';
  *                                               parsed (defaults resolve in `start()`). Objects
  *                                               strictly under it are stored inline in the
  *                                               metadata store; `0`/absent disables tiering.
+ * @param {number}           [conf.sweepGrace] - How old a temp file must be before the
+ *                                               build-time orphan sweep reclaims it, in ms
+ *                                               (defaults resolve in `start()`). Same key,
+ *                                               default and meaning as cas's; `sharded` has no
+ *                                               periodic sweep, so this gates the build pass
+ *                                               alone (#B349).
  * @param {StorageMetaStore} metaStore         - Metadata backend (embedded SQLite by default).
  * @returns {StorageDriver} A ready driver.
  *
@@ -119,6 +139,38 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
     var inlineThreshold = ( typeof conf.inlineThreshold === 'number' && conf.inlineThreshold > 0 )
         ? conf.inlineThreshold
         : 0;
+    var sweepGraceMs = ( typeof conf.sweepGrace === 'number' && conf.sweepGrace > 0 )
+        ? conf.sweepGrace
+        : 0;
+
+    // #B349 — build-time temp-orphan sweep, age-gated. A put() whose PROCESS
+    // died leaves its temp behind, and until now nothing on `sharded` ever
+    // removed it: cas has this same pass and `stream` has its own, so sharded
+    // was the one strategy where the leak was permanent. Build time is the
+    // right and only trigger — a restart is the earliest moment the previous
+    // process is provably gone — and the age gate is what keeps a SIBLING
+    // process's in-flight write safe on a shared root, which is precisely the
+    // shape a connector-backed store exists to support. Never throws: an
+    // unreadable temp dir is the next put()'s problem to report properly, and
+    // a sweep that took the build down would be a worse failure than the leak
+    // it prevents.
+    if ( sweepGraceMs > 0 ) {
+        try {
+            var _tmpD = nodePath.join(root, TMP_DIR);
+            if ( fs.existsSync(_tmpD) ) {
+                var _cutoff  = Date.now() - sweepGraceMs;
+                var _entries = fs.readdirSync(_tmpD);
+                for (var _t = 0; _t < _entries.length; _t++) {
+                    var _tp = nodePath.join(_tmpD, _entries[_t]);
+                    try {
+                        if ( fs.statSync(_tp).mtimeMs < _cutoff ) {
+                            fs.unlinkSync(_tp);
+                        }
+                    } catch (_e) { /* raced or unreadable — leave it */ }
+                }
+            }
+        } catch (_e) { /* never blocks the build */ }
+    }
 
     /**
      * Resolve a caller-supplied key to a confined absolute path.
@@ -143,7 +195,7 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
      */
     var capabilities = {
         offload   : false,
-        ranges    : false,
+        ranges    : true,
         dedup     : false,
         resumable : false,
         inline    : ( inlineThreshold > 0 )
@@ -232,14 +284,42 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
                 }
             }
 
+            /**
+             * Abandon the put: tear both streams down, remove the temp, report.
+             *
+             * #B358 — the unlink WAITS for the write stream to close, and the
+             * callback waits with it. `createWriteStream()` opens
+             * asynchronously, so an error arriving right after `spill()` — the
+             * source failing on the chunk that crossed `inlineThreshold` is the
+             * measured case — reaches here while the `open(2)` is still in the
+             * threadpool: the temp does not exist YET at this instant but lands
+             * a tick later, on the worker thread, with no event-loop turn in
+             * between for a synchronous check to see. The previous
+             * `existsSync`-gated unlink therefore read false, skipped, and
+             * orphaned the temp. A null `ws` means `spill()` never ran and no
+             * temp was ever created, so that arm finishes immediately; `force`
+             * makes the unlink a no-op there rather than a special case.
+             *
+             * @inner
+             * @param {*} err - The failure to report; wrapped if not an Error.
+             * @returns {void}
+             */
             var onError = function(err) {
                 if (settled) return;
                 settled = true;
                 head = null;
                 try { stream.destroy(); } catch (_e) {}
-                if (ws) { try { ws.destroy(); } catch (_e) {} }
-                try { if ( fs.existsSync(tmp) ) fs.unlinkSync(tmp); } catch (_e) {}
-                fn(( err instanceof Error ) ? err : new Error(String(err)));
+
+                var finish = function() {
+                    try { fs.rmSync(tmp, { force: true }); } catch (_e) {}
+                    fn(( err instanceof Error ) ? err : new Error(String(err)));
+                };
+
+                // No ws (never spilled), or one that already emitted 'close' —
+                // neither will emit it again, so those arms must not wait.
+                if ( !ws || ws.closed || ws.destroyed ) { return finish(); }
+                ws.once('close', finish);
+                try { ws.destroy(); } catch (_e) { finish(); }
             };
 
             /**
@@ -404,7 +484,7 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
                     return fn(null, Readable.from([m.data]));
                 }
                 if ( !fs.existsSync(r.path) ) {
-                    return fn(new Error('[storage:' + name + '] no object for key `' + key + '`'));
+                    return fn(util.codedError('STORAGE_NO_OBJECT', '[storage:' + name + '] no object for key `' + key + '`'));
                 }
                 var rs = fs.createReadStream(r.path);
                 // Armed before the caller can touch it (#B143) — an fs error after
@@ -423,6 +503,83 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
             });
         },
 
+        /**
+         * Byte-range read — the driver half of HTTP Range serving.
+         *
+         * `end` is INCLUSIVE, matching both the HTTP `Range` header and
+         * `fs.createReadStream({start, end})`, so no arithmetic happens at the
+         * boundary between them. An `end` past the last byte is CLAMPED rather
+         * than refused (RFC 9110: a range whose start is satisfiable is
+         * satisfiable); only a `start` at or beyond the object's size is
+         * unsatisfiable, and that is the caller's 416.
+         *
+         * Both tiers answer: an inline row slices its buffer, a file-backed row
+         * gets a positioned read stream. The discriminator is `m.data != null`,
+         * exactly as `get()`/`resolve()` use — never the configured threshold,
+         * so a threshold change stays retroactively safe here too.
+         *
+         * @param {string}   key   - The opaque storage key.
+         * @param {number}   start - First byte offset, inclusive; integer >= 0.
+         * @param {number}   end   - Last byte offset, INCLUSIVE; integer >= start.
+         * @param {function} fn    - `fn(err, stream)`.
+         * @returns {void}
+         *
+         * @example
+         * // Serving `Range: bytes=0-1023`
+         * driver.getRange(key, 0, 1023, function (err, stream) {
+         *     if (err) { return self.throwError(416); }
+         *     stream.pipe(res);
+         * });
+         */
+        getRange: function(key, start, end, fn) {
+            if ( typeof fn !== 'function' ) {
+                throw new Error('[storage:' + name + '] getRange() requires a callback');
+            }
+            if ( !Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end < start ) {
+                return fn(util.codedError('STORAGE_INVALID_RANGE', '[storage:' + name + '] getRange(): invalid range [' + start + ', ' + end + '] — both bounds must be integers with 0 <= start <= end'));
+            }
+            var r = resolvePath(key);
+            if ( r.error ) { return fn(r.error); }
+            metaStore.get(key, function(metaErr, m) {
+                if (metaErr) { return fn(metaErr); }
+                if ( m && m.data != null ) {
+                    if ( start >= m.data.length ) {
+                        return fn(util.codedError('STORAGE_RANGE_UNSATISFIABLE', '[storage:' + name + '] getRange(): start ' + start + ' is beyond the object size (' + m.data.length + ') for key `' + key + '`'));
+                    }
+                    // subarray() is a VIEW, no copy; the array wrap is the same
+                    // load-bearing detail as in get() — Readable.from(buffer)
+                    // would iterate the bytes as INTEGERS.
+                    return fn(null, Readable.from([ m.data.subarray(start, Math.min(end, m.data.length - 1) + 1) ]));
+                }
+                if ( !fs.existsSync(r.path) ) {
+                    return fn(util.codedError('STORAGE_NO_OBJECT', '[storage:' + name + '] no object for key `' + key + '`'));
+                }
+                var size = fs.statSync(r.path).size;
+                if ( start >= size ) {
+                    return fn(util.codedError('STORAGE_RANGE_UNSATISFIABLE', '[storage:' + name + '] getRange(): start ' + start + ' is beyond the object size (' + size + ') for key `' + key + '`'));
+                }
+                // The clamp is EXPLICITNESS, not necessity: measured, both
+                // `createReadStream({end})` and `Buffer.subarray()` already stop
+                // at the last byte, so the suite's clamp test passes with or
+                // without it. Stated here so the contract does not rest on two
+                // different APIs each happening to do the right thing.
+                var rs = fs.createReadStream(r.path, { start: start, end: Math.min(end, size - 1) });
+                // Armed before the caller can touch it (#B143) — identical
+                // handoff discipline to get().
+                var handed = false;
+                rs.on('error', function(err) {
+                    if (handed) return;
+                    handed = true;
+                    fn(err);
+                });
+                rs.on('open', function() {
+                    if (handed) return;
+                    handed = true;
+                    fn(null, rs);
+                });
+            });
+        },
+
         /**
          * Metadata for `key`.
          *
@@ -500,8 +657,11 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
          * arrives with the s3 adapter. Callers must branch on `kind` rather
          * than assume a path is always available.
          *
-         * @param {string}   key - The opaque storage key.
-         * @param {function} fn  - `fn(err, {kind: 'path', path: string} | {kind: 'inline'})`.
+         * @param {string}   key    - The opaque storage key.
+         * @param {?object}  [opts] - Offload response overrides — consumed by
+         *                            offload-capable adapters (`s3`); accepted
+         *                            and IGNORED here.
+         * @param {function} fn     - `fn(err, {kind: 'path', path: string} | {kind: 'inline'})`.
          * @returns {void}
          *
          * @example
@@ -512,7 +672,8 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
          *     driver.get(key, function (err, stream) { stream.pipe(res); });
          * });
          */
-        resolve: function(key, fn) {
+        resolve: function(key, opts, fn) {
+            if ( typeof opts === 'function' ) { fn = opts; }
             if ( typeof fn !== 'function' ) {
                 throw new Error('[storage:' + name + '] resolve() requires a callback');
             }
@@ -524,7 +685,7 @@ module.exports = function createLocalDriver(name, conf, metaStore) {
                     return fn(null, { kind: 'inline' });
                 }
                 if ( !fs.existsSync(r.path) ) {
-                    return fn(new Error('[storage:' + name + '] no object for key `' + key + '`'));
+                    return fn(util.codedError('STORAGE_NO_OBJECT', '[storage:' + name + '] no object for key `' + key + '`'));
                 }
                 fn(null, { kind: 'path', path: r.path });
             });