gina 0.6.7 → 0.6.8

package/framework/{v0.6.7 → v0.6.8}/lib/storage/src/main.js

@@ -15,8 +15,10 @@
  * strategy; the tiering slice adds size tiering (sub-threshold objects live
  * inline in the metadata store — see `inlineThreshold`); the cas slice adds
  * the `cas` strategy (content-addressed, refcounted, GC-swept — see
- * `src/local-cas`); `stream` and `s3` arrive with the demand that needs them,
- * each capability-gated.
+ * `src/local-cas`); the stream slice adds the `stream` strategy (large
+ * sequential media with resumable, out-of-order segment uploads — see
+ * `src/local-stream`); `s3` arrives with the demand that needs it, each
+ * capability-gated.
  *
  * **Keys are opaque.** A caller stores a stream and receives a key; it must
  * never parse, compose or assume anything about that key's shape. That is what
@@ -112,6 +114,15 @@ var createLocalDriver = require('./local');
  */
 var createLocalCasDriver = require('./local-cas');
 
+/**
+ * The `local` adapter / `stream` strategy pair — large sequential media with
+ * resumable, out-of-order segment uploads.
+ *
+ * @inner
+ * @type {function(string, object, object): StorageDriver}
+ */
+var createLocalStreamDriver = require('./local-stream');
+
 /**
  * Strategy name → driver factory. THE dispatch seam: `start()` builds each
  * driver through this map (strategy names are validated before `start()`
@@ -124,7 +135,37 @@ var createLocalCasDriver = require('./local-cas');
  */
 var FACTORIES = {
     sharded : createLocalDriver,
-    cas     : createLocalCasDriver
+    cas     : createLocalCasDriver,
+    stream  : createLocalStreamDriver
+};
+
+/**
+ * `s3` adapter factory — object storage on any S3-compatible provider,
+ * storeless (the provider owns metadata). The SDK stays the consuming
+ * project's dependency, resolved through `start()`'s injected
+ * `requireProjectModule` (lib/storage is framework-independent, so the
+ * resolver is built by the caller where the path globals are legal).
+ *
+ * @inner
+ * @type {function(string, object, ?object, object): StorageDriver}
+ */
+var createS3Driver = require('./s3');
+
+/**
+ * Adapter × strategy — THE two-axis dispatch the strategy map above always
+ * said would arrive with a second adapter. Expressed as COMPOSITION rather
+ * than a rekey: `FACTORIES` keeps its flat strategy-keyed shape because the
+ * `storage:*` CLI's offline path dispatches through it directly (and its
+ * source is pinned by the maintenance suite) — offline builds are local-only
+ * by construction, so the flat map IS the local adapter's row here.
+ *
+ * @inner
+ * @constant
+ * @type {Object.<string, Object.<string, function>>}
+ */
+var ADAPTER_FACTORIES = {
+    local : FACTORIES,
+    s3    : { sharded: createS3Driver }
 };
 
 /**
@@ -146,7 +187,7 @@ var createEmbeddedMetaStore = require('./meta-store');
  * @constant
  * @type {string[]}
  */
-var ADAPTERS = ['local'];
+var ADAPTERS = ['local', 's3'];
 
 /**
  * Strategies implemented today.
@@ -155,18 +196,26 @@ var ADAPTERS = ['local'];
  * @constant
  * @type {string[]}
  */
-var STRATEGIES = ['sharded', 'cas'];
+var STRATEGIES = ['sharded', 'cas', 'stream'];
 
 /**
  * Strategies named in the design but not yet implemented. Kept distinct from
  * "unknown" so the boot message can say *deferred* instead of *typo* — an
- * operator who wrote `stream` made a scheduling mistake, not a spelling one.
+ * operator who wrote a designed-but-unshipped name made a scheduling mistake,
+ * not a spelling one.
+ *
+ * **Currently EMPTY**: `cas` left this list when the cas slice shipped, and
+ * `stream` when the stream slice did (`s3` arrived as an ADAPTER — the
+ * two-axis dispatch below — so it never passed through here). The list and
+ * its boot branch stay — cheap, and the next designed-before-implemented
+ * strategy inherits a working message instead of re-deriving one. Until then
+ * the branch is exercised by nothing, which is recorded rather than hidden.
  *
  * @inner
  * @constant
  * @type {string[]}
  */
-var DEFERRED_STRATEGIES = ['stream'];
+var DEFERRED_STRATEGIES = [];
 
 /**
  * Per-driver keys, BY STRATEGY. Anything else in a driver block is reported
@@ -182,10 +231,35 @@ var DEFERRED_STRATEGIES = ['stream'];
  * @type {Object.<string, string[]>}
  */
 var STRATEGY_KEYS = {
-    sharded : ['adapter', 'strategy', 'root', 'maxObjectSize', 'store', 'inlineThreshold'],
-    cas     : ['adapter', 'strategy', 'root', 'maxObjectSize', 'store', 'inlineThreshold', 'hash', 'fsync', 'sweepInterval', 'sweepGrace']
+    // NB `sharded` consumes `sweepGrace` but NOT `sweepInterval`: it reclaims
+    // crashed-put temp orphans at BUILD only, and has no periodic machinery to
+    // hang a tick on. A restart is also the earliest moment the previous
+    // process is provably gone, which is the whole safety argument (#B349).
+    sharded : ['adapter', 'strategy', 'root', 'maxObjectSize', 'store', 'inlineThreshold', 'sweepGrace'],
+    cas     : ['adapter', 'strategy', 'root', 'maxObjectSize', 'store', 'inlineThreshold', 'hash', 'fsync', 'sweepInterval', 'sweepGrace'],
+    // NB `stream` carries NEITHER `inlineThreshold` NOR `hash`, deliberately:
+    // a multi-gigabyte strategy has no business inlining its payload into a
+    // metadata row, and its write path does not hash. Accepting either key
+    // would be exactly the "configured behaviour that never happens" lie this
+    // whole warning exists to prevent, so both warn as ignored instead.
+    stream  : ['adapter', 'strategy', 'root', 'maxObjectSize', 'store', 'chunkSize', 'fsync', 'sessionTtl', 'sessionSweepInterval']
 };
 
+/**
+ * Per-driver keys for the `s3` ADAPTER (one list — s3 runs a single
+ * strategy, so the per-strategy split above does not apply). NB what is
+ * deliberately ABSENT: `root` (no filesystem), `store`/`inlineThreshold`
+ * (storeless — no metadata rows to inline into), `hash`/`fsync` (the
+ * provider owns integrity and durability). Each warns as ignored — the
+ * same "configured behaviour that never happens" honesty as the strategy
+ * lists.
+ *
+ * @inner
+ * @constant
+ * @type {string[]}
+ */
+var S3_KEYS = ['adapter', 'strategy', 'bucket', 'region', 'endpoint', 'prefix', 'accessKeyId', 'secretAccessKey', 'sessionToken', 'forcePathStyle', 'presignExpiry', 'maxObjectSize', 'sweepGrace'];
+
 /**
  * Default per-object byte ceiling when a driver sets no `maxObjectSize`.
  * Defense in depth only: no per-file cap exists anywhere upstream today (the
@@ -217,6 +291,20 @@ var DEFAULT_MAX_OBJECT_SIZE = 100 * 1024 * 1024;
  */
 var DEFAULT_INLINE_THRESHOLD = 64 * 1024;
 
+/**
+ * Default presigned-URL lifetime for `s3` drivers when `presignExpiry` is
+ * unset. 15 minutes: long enough for a redirect chain and a slow download to
+ * START (a transfer in flight survives expiry — S3 validates the signature
+ * at request time), short enough that a leaked URL goes stale fast. The
+ * serving facade re-presigns per request, so the lifetime is not a caching
+ * knob.
+ *
+ * @inner
+ * @constant
+ * @type {number}
+ */
+var DEFAULT_PRESIGN_EXPIRY = 15 * 60 * 1000;
+
 /**
  * Default digest algorithm for `cas` drivers. SHA-256: hardware-accelerated
  * on every deployment target (SHA-NI, ARMv8 crypto extensions put it around
@@ -269,6 +357,64 @@ var DEFAULT_SWEEP_INTERVAL = 15 * 60 * 1000;
  */
 var DEFAULT_SWEEP_GRACE = 60 * 60 * 1000;
 
+/**
+ * Default segment size for `stream` drivers, in bytes. It becomes the write
+ * stream's `highWaterMark` — the same syscall size with no hand-rolled buffer
+ * and no extra copy, so memory stays bounded by the stream machinery rather
+ * than by us — and is echoed to clients as the size the write path is tuned
+ * for. 8MB: large enough that per-segment overheads (an fsync, a marker
+ * create) disappear against the transfer, small enough that re-sending one
+ * after a broken connection is cheap.
+ *
+ * REASONED against a measured mechanism, NOT tuned to a workload: no consumer
+ * with a large-sequential-media workload exists yet, so re-measure when one
+ * appears.
+ *
+ * @inner
+ * @constant
+ * @type {number}
+ */
+var DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024;
+
+/**
+ * Whether `stream` publishes and segment writes fsync by default. `true` —
+ * the durability contract of a RESUMABLE upload is that a range the client
+ * was told is durable actually is, and the cost is ~2% of network-paced
+ * ingest at 100 Mbps (measured; ~19% at 1 Gbps, fsync-dominant at 10 Gbps).
+ * A LAN-ingest consumer sets `fsync: false` and accepts the documented
+ * power-loss window.
+ *
+ * @inner
+ * @constant
+ * @type {boolean}
+ */
+var DEFAULT_STREAM_FSYNC = true;
+
+/**
+ * Default lifetime of an untouched resumable upload session for `stream`
+ * drivers, in ms. REASONED, not benchmarked: it must comfortably exceed a
+ * paused-then-resumed upload over a bad link, and a session holds real disk,
+ * so 24 hours. A zero TTL would eat in-flight sessions and snaps back to this
+ * default (the `sweepGrace` reasoning).
+ *
+ * @inner
+ * @constant
+ * @type {number}
+ */
+var DEFAULT_SESSION_TTL = 24 * 60 * 60 * 1000;
+
+/**
+ * Default period of the abandoned-session sweep for `stream` drivers, in ms.
+ * REASONED, not benchmarked: the sweep gates reclaim latency only — no hot
+ * path — so the default just bounds how long an abandoned session's disk
+ * lingers past its TTL. 1 hour.
+ *
+ * @inner
+ * @constant
+ * @type {number}
+ */
+var DEFAULT_SESSION_SWEEP_INTERVAL = 60 * 60 * 1000;
+
 /**
  * Built drivers, keyed by name. `null` until {@link start} runs — deliberately
  * NOT built at require time, because `test/lib/types-runtime-parity.test.js`
@@ -326,6 +472,12 @@ var _default = null;
  *     (which would re-open the sweep-vs-put race) — each falls back to its
  *     default; `'0s'` is legal and silent for `sweepInterval` only (periodic
  *     sweep off);
+ *   - a stream `chunkSize` that is not a unit-suffixed size (or is zero), a
+ *     `fsync` that is not a boolean, a `sessionTtl`/`sessionSweepInterval`
+ *     that is not a unit-suffixed duration, or a zero `sessionTtl` (which
+ *     would reclaim in-flight uploads) — each falls back to its default;
+ *     `'0s'` is legal and silent for `sessionSweepInterval` only (periodic
+ *     session sweep off, the build-time pass still runs);
  *   - driver keys the selected strategy does not consume — they are ignored,
  *     and a silently-ignored key reads as configured behaviour that never
  *     happens (the list is per-strategy: `fsync` is consumed by `cas` and
@@ -426,6 +578,105 @@ function validateConfig(storageBlock, context) {
             return out;
         }
 
+        // ─── `s3` — a SELF-CONTAINED branch, so the local checks below stay
+        // byte-identical. On external object storage the layout algorithm
+        // collapses to key naming, the filesystem keys (`root`, `store`,
+        // tiering, fsync) have no meaning, and the identity keys are the
+        // provider coordinates instead.
+        if ( d.adapter === 's3' ) {
+            if ( typeof(d.strategy) != 'undefined' && d.strategy !== null && d.strategy !== 'sharded' ) {
+                if ( d.strategy === 'cas' || d.strategy === 'stream' ) {
+                    out.fatal = 'driver `' + name + '`: the `' + d.strategy + '` strategy does not run on the `s3` adapter — the provider owns placement: S3 ETags are not content digests (no cas dedup), and multipart is the provider\'s own resumable (no stream sessions). Use `sharded` (or omit `strategy`, it is the s3 default), or move the driver to the `local` adapter.';
+                    return out;
+                }
+                out.fatal = 'driver `' + name + '`: unknown strategy ' + JSON.stringify(d.strategy) + ' — the `s3` adapter supports `sharded` only, and defaults to it when `strategy` is omitted';
+                return out;
+            }
+
+            if ( typeof(d.bucket) != 'string' || d.bucket.length === 0 ) {
+                out.fatal = 'driver `' + name + '`: `bucket` must be a non-empty bucket name for the `s3` adapter';
+                return out;
+            }
+            if ( /\$\{/.test(d.bucket) ) {
+                out.fatal = 'driver `' + name + '`: `bucket` carries an unresolved `${...}` placeholder (' + d.bucket + ') — only the anchored `${secret:KEY}` form is substituted at config load';
+                return out;
+            }
+
+            if ( typeof(d.prefix) != 'undefined' && d.prefix !== null && d.prefix !== '' ) {
+                if ( typeof(d.prefix) != 'string' ) {
+                    out.fatal = 'driver `' + name + '`: `prefix` must be a string key prefix';
+                    return out;
+                }
+                if ( /\$\{/.test(d.prefix) ) {
+                    out.fatal = 'driver `' + name + '`: `prefix` carries an unresolved `${...}` placeholder (' + d.prefix + ') — only the anchored `${secret:KEY}` form is substituted at config load';
+                    return out;
+                }
+                if ( d.prefix.charAt(0) === '/' || d.prefix.indexOf('\\') > -1 || d.prefix.indexOf('..') > -1 ) {
+                    out.fatal = 'driver `' + name + '`: `prefix` must be a relative key prefix (no leading `/`, no `\\`, no `..`) — got `' + d.prefix + '`';
+                    return out;
+                }
+            }
+
+            if ( typeof(d.endpoint) != 'undefined' && d.endpoint !== null && ( typeof(d.endpoint) != 'string' || d.endpoint.length === 0 ) ) {
+                out.fatal = 'driver `' + name + '`: `endpoint` must be a non-empty S3-compatible endpoint (e.g. "s3.fr-par.scw.cloud")';
+                return out;
+            }
+            if ( typeof(d.region) != 'undefined' && d.region !== null && ( typeof(d.region) != 'string' || d.region.length === 0 ) ) {
+                out.fatal = 'driver `' + name + '`: `region` must be a non-empty region name';
+                return out;
+            }
+
+            // static credentials are BOTH-OR-NEITHER: a half pair is almost
+            // certainly a mistake, but the SDK default provider chain
+            // (env/IMDS/IRSA) is a legitimate fallback, so this warns rather
+            // than refusing the boot.
+            var hasAk = ( typeof(d.accessKeyId) == 'string' && d.accessKeyId.length > 0 );
+            var hasSk = ( typeof(d.secretAccessKey) == 'string' && d.secretAccessKey.length > 0 );
+            if ( hasAk !== hasSk ) {
+                out.warnings.push('driver `' + name + '`: only one of `accessKeyId`/`secretAccessKey` is set — static credentials need BOTH; falling back to the SDK default provider chain (env / shared config / IMDS / IRSA)');
+            } else if ( !hasAk && typeof(d.sessionToken) == 'string' && d.sessionToken.length > 0 ) {
+                out.warnings.push('driver `' + name + '`: `sessionToken` is set without `accessKeyId`/`secretAccessKey` — it is ignored; the SDK default provider chain supplies its own session credentials');
+            }
+
+            if ( typeof(d.forcePathStyle) != 'undefined' && d.forcePathStyle !== null && typeof(d.forcePathStyle) != 'boolean' ) {
+                out.warnings.push('driver `' + name + '`: `forcePathStyle` must be a boolean — got ' + JSON.stringify(d.forcePathStyle) + '; using the default (false)');
+            }
+
+            if ( typeof(d.presignExpiry) != 'undefined' && d.presignExpiry !== null ) {
+                if ( isNaN(util.parseDuration(d.presignExpiry)) ) {
+                    out.warnings.push('driver `' + name + '`: `presignExpiry` must carry a unit (ms/s/m/h/d, e.g. "15m") — got ' + JSON.stringify(d.presignExpiry) + '; using the default (' + DEFAULT_PRESIGN_EXPIRY + ' ms)');
+                } else if ( util.parseDuration(d.presignExpiry) <= 0 ) {
+                    out.warnings.push('driver `' + name + '`: `presignExpiry` must be greater than zero — got ' + JSON.stringify(d.presignExpiry) + '; using the default (' + DEFAULT_PRESIGN_EXPIRY + ' ms)');
+                }
+            }
+
+            // same key, same default as the local strategies' temp-orphan
+            // gate — here it ages the multipart-orphan sweep (an incomplete
+            // multipart upload bills until aborted).
+            if ( typeof(d.sweepGrace) != 'undefined' && d.sweepGrace !== null ) {
+                if ( isNaN(util.parseDuration(d.sweepGrace)) ) {
+                    out.warnings.push('driver `' + name + '`: `sweepGrace` must carry a unit (ms/s/m/h/d, e.g. "1h") — got ' + JSON.stringify(d.sweepGrace) + '; using the default (' + DEFAULT_SWEEP_GRACE + ' ms)');
+                } else if ( util.parseDuration(d.sweepGrace) <= 0 ) {
+                    out.warnings.push('driver `' + name + '`: `sweepGrace` must be greater than zero — got ' + JSON.stringify(d.sweepGrace) + '; using the default (' + DEFAULT_SWEEP_GRACE + ' ms)');
+                }
+            }
+
+            if ( typeof(d.maxObjectSize) != 'undefined' && d.maxObjectSize !== null ) {
+                if ( isNaN(util.parseSize(d.maxObjectSize)) ) {
+                    out.warnings.push('driver `' + name + '`: `maxObjectSize` must carry a unit (B/KB/MB/GB, e.g. "50MB") — got ' + JSON.stringify(d.maxObjectSize) + '; using the default (' + DEFAULT_MAX_OBJECT_SIZE + ' bytes)');
+                } else if ( util.parseSize(d.maxObjectSize) <= 0 ) {
+                    out.warnings.push('driver `' + name + '`: `maxObjectSize` must be greater than zero — got ' + JSON.stringify(d.maxObjectSize) + '; using the default (' + DEFAULT_MAX_OBJECT_SIZE + ' bytes)');
+                }
+            }
+
+            var s3extra = Object.keys(d).filter(function (k) { return S3_KEYS.indexOf(k) < 0; });
+            if ( s3extra.length > 0 ) {
+                out.warnings.push('driver `' + name + '`: key(s) ' + s3extra.join(', ') + ' are not used by the `s3` adapter and are ignored');
+            }
+
+            continue;
+        }
+
         if ( DEFERRED_STRATEGIES.indexOf(d.strategy) > -1 ) {
             out.fatal = 'driver `' + name + '`: the `' + d.strategy + '` strategy is designed but not implemented yet — use `' + STRATEGIES.join('|') + '` for now';
             return out;
@@ -514,17 +765,63 @@ function validateConfig(storageBlock, context) {
                     out.warnings.push('driver `' + name + '`: `sweepInterval` must carry a unit (ms/s/m/h/d, e.g. "15m"), or be "0s" to disable the periodic sweep — got ' + JSON.stringify(d.sweepInterval) + '; using the default (' + DEFAULT_SWEEP_INTERVAL + ' ms)');
                 }
             }
+        }
+
+        // `sweepGrace` is shared, so its lint is too: under `cas` it gates the
+        // blob GC AND the temp-orphan cutoff, under `sharded` the temp-orphan
+        // cutoff alone (#B349). `sweepInterval` deliberately stays cas-only —
+        // sharded sweeps at build, never on a tick.
+        if ( d.strategy === 'cas' || d.strategy === 'sharded' ) {
             if ( typeof(d.sweepGrace) != 'undefined' && d.sweepGrace !== null ) {
                 if ( isNaN(util.parseDuration(d.sweepGrace)) ) {
                     out.warnings.push('driver `' + name + '`: `sweepGrace` must carry a unit (ms/s/m/h/d, e.g. "1h") — got ' + JSON.stringify(d.sweepGrace) + '; using the default (' + DEFAULT_SWEEP_GRACE + ' ms)');
                 } else if ( util.parseDuration(d.sweepGrace) <= 0 ) {
-                    // grace 0 would let the sweep race an in-flight identical
-                    // put — the exact window the grace exists to close
+                    // grace 0 would let the sweep eat a temp still being
+                    // written — under cas it would also race an in-flight
+                    // identical put. Either way it re-opens the exact window
+                    // the grace exists to close.
                     out.warnings.push('driver `' + name + '`: `sweepGrace` must be greater than zero — got ' + JSON.stringify(d.sweepGrace) + '; using the default (' + DEFAULT_SWEEP_GRACE + ' ms)');
                 }
             }
         }
 
+        if ( d.strategy === 'stream' ) {
+            // `chunkSize` is a size key and takes the size parser's strictness
+            // (a unit is required); it only tunes the write stream's
+            // highWaterMark, so an unusable value degrades to the default
+            // rather than refusing the boot.
+            if ( typeof(d.chunkSize) != 'undefined' && d.chunkSize !== null ) {
+                if ( isNaN(util.parseSize(d.chunkSize)) ) {
+                    out.warnings.push('driver `' + name + '`: `chunkSize` must carry a unit (B/KB/MB/GB, e.g. "8MB") — got ' + JSON.stringify(d.chunkSize) + '; using the default (' + DEFAULT_CHUNK_SIZE + ' bytes)');
+                } else if ( util.parseSize(d.chunkSize) <= 0 ) {
+                    out.warnings.push('driver `' + name + '`: `chunkSize` must be greater than zero — got ' + JSON.stringify(d.chunkSize) + '; using the default (' + DEFAULT_CHUNK_SIZE + ' bytes)');
+                }
+            }
+
+            if ( typeof(d.fsync) != 'undefined' && d.fsync !== null && typeof(d.fsync) != 'boolean' ) {
+                out.warnings.push('driver `' + name + '`: `fsync` must be a boolean — got ' + JSON.stringify(d.fsync) + '; using the default (' + DEFAULT_STREAM_FSYNC + ')');
+            }
+
+            // duration keys, same strictness as the cas pair: a unit is
+            // required, because `'15m'` means minutes here and megabytes in a
+            // size key and neither parser guesses.
+            if ( typeof(d.sessionTtl) != 'undefined' && d.sessionTtl !== null ) {
+                if ( isNaN(util.parseDuration(d.sessionTtl)) ) {
+                    out.warnings.push('driver `' + name + '`: `sessionTtl` must carry a unit (ms/s/m/h/d, e.g. "24h") — got ' + JSON.stringify(d.sessionTtl) + '; using the default (' + DEFAULT_SESSION_TTL + ' ms)');
+                } else if ( util.parseDuration(d.sessionTtl) <= 0 ) {
+                    // a zero TTL would reclaim sessions that are still being
+                    // written to — the same class of foot-gun as a zero
+                    // sweepGrace, so it never means "off"
+                    out.warnings.push('driver `' + name + '`: `sessionTtl` must be greater than zero — a zero TTL would reclaim in-flight uploads; got ' + JSON.stringify(d.sessionTtl) + '; using the default (' + DEFAULT_SESSION_TTL + ' ms)');
+                }
+            }
+            if ( typeof(d.sessionSweepInterval) != 'undefined' && d.sessionSweepInterval !== null ) {
+                if ( isNaN(util.parseDuration(d.sessionSweepInterval)) ) {
+                    out.warnings.push('driver `' + name + '`: `sessionSweepInterval` must carry a unit (ms/s/m/h/d, e.g. "1h"), or be "0s" to disable the periodic sweep — got ' + JSON.stringify(d.sessionSweepInterval) + '; using the default (' + DEFAULT_SESSION_SWEEP_INTERVAL + ' ms)');
+                }
+            }
+        }
+
         var consumed = STRATEGY_KEYS[d.strategy];
         var extra = Object.keys(d).filter(function (k) { return consumed.indexOf(k) < 0; });
         if ( extra.length > 0 ) {
@@ -597,6 +894,50 @@ function resolveDriverConf(conf) {
         inlineThreshold : inline
     };
 
+    // s3 resolves here and RETURNS EARLY — none of the local strategy blocks
+    // below apply. `strategy` collapses to key naming (`sharded`, the only
+    // value and the default); `prefix`/`endpoint` are normalised so the
+    // factory stays dumb; `region` falls back to the S3-compat convention
+    // (`us-east-1`) ONLY when a custom endpoint is configured — real AWS
+    // keeps the SDK's own resolution chain, whose "missing region" message
+    // beats a silently mis-signed default.
+    if ( conf.adapter === 's3' ) {
+        resolved.strategy = 'sharded';
+        resolved.bucket   = conf.bucket;
+        var pfx = ( typeof conf.prefix === 'string' ) ? conf.prefix : '';
+        if ( pfx.length && pfx.charAt(pfx.length - 1) !== '/' ) { pfx += '/'; }
+        resolved.prefix = pfx;
+        if ( typeof conf.endpoint === 'string' && conf.endpoint.length ) {
+            resolved.endpoint = /^https?:\/\//i.test(conf.endpoint) ? conf.endpoint : ('https://' + conf.endpoint);
+        }
+        resolved.region = ( typeof conf.region === 'string' && conf.region.length )
+            ? conf.region
+            : ( resolved.endpoint ? 'us-east-1' : undefined );
+        if ( conf.forcePathStyle === true ) { resolved.forcePathStyle = true; }
+        if ( typeof conf.accessKeyId === 'string' && conf.accessKeyId.length
+            && typeof conf.secretAccessKey === 'string' && conf.secretAccessKey.length ) {
+            resolved.accessKeyId     = conf.accessKeyId;
+            resolved.secretAccessKey = conf.secretAccessKey;
+            if ( typeof conf.sessionToken === 'string' && conf.sessionToken.length ) {
+                resolved.sessionToken = conf.sessionToken;
+            }
+        }
+        var pe = util.parseDuration(conf.presignExpiry);
+        resolved.presignExpiry = ( isNaN(pe) || pe <= 0 ) ? DEFAULT_PRESIGN_EXPIRY : pe;
+        var s3g = util.parseDuration(conf.sweepGrace);
+        resolved.sweepGrace = ( isNaN(s3g) || s3g <= 0 ) ? DEFAULT_SWEEP_GRACE : s3g;
+        return resolved;
+    }
+
+    // sharded takes `sweepGrace` alone — the age gate on its build-time
+    // temp-orphan sweep (#B349). Same key, same default and same meaning as
+    // cas's ("how long before an unfinished temp is presumed abandoned"), but
+    // no `sweepInterval`: there is no periodic pass to configure.
+    if ( conf.strategy === 'sharded' ) {
+        var tsg = util.parseDuration(conf.sweepGrace);
+        resolved.sweepGrace = ( isNaN(tsg) || tsg <= 0 ) ? DEFAULT_SWEEP_GRACE : tsg;
+    }
+
     // cas defaults resolve here too, same pattern. `hash` was validated
     // against crypto.getHashes() by validateConfig; lowercasing makes the
     // key's algorithm segment canonical.
@@ -611,6 +952,20 @@ function resolveDriverConf(conf) {
         resolved.sweepGrace = ( isNaN(sg) || sg <= 0 ) ? DEFAULT_SWEEP_GRACE : sg;      // 0 grace would re-open the sweep race
     }
 
+    // stream defaults, same pattern. NB `inlineThreshold` is resolved above
+    // for every strategy but the stream factory ignores it — the key is not in
+    // STRATEGY_KEYS.stream, so setting it warns as ignored, and
+    // capabilities.inline stays false.
+    if ( conf.strategy === 'stream' ) {
+        var cs = util.parseSize(conf.chunkSize);
+        resolved.chunkSize = ( isNaN(cs) || cs <= 0 ) ? DEFAULT_CHUNK_SIZE : cs;
+        resolved.fsync = ( typeof conf.fsync === 'boolean' ) ? conf.fsync : DEFAULT_STREAM_FSYNC;
+        var st = util.parseDuration(conf.sessionTtl);
+        resolved.sessionTtl = ( isNaN(st) || st <= 0 ) ? DEFAULT_SESSION_TTL : st;      // 0 would reclaim in-flight uploads
+        var ssi = util.parseDuration(conf.sessionSweepInterval);
+        resolved.sessionSweepInterval = ( isNaN(ssi) || ssi < 0 ) ? DEFAULT_SESSION_SWEEP_INTERVAL : ssi; // '0s' is legal: periodic sweep off
+    }
+
     return resolved;
 }
 
@@ -628,7 +983,15 @@ function resolveDriverConf(conf) {
  * @param {object}   [opt.stores]      - Pre-built connector metadata stores keyed by driver
  *                                       name (the caller resolves them through
  *                                       `lib/storage-store`). A driver absent from this map
- *                                       gets the embedded SQLite store inside its own root.
+ *                                       gets the embedded SQLite store inside its own root
+ *                                       (`s3` drivers get neither — storeless).
+ * @param {function} [opt.requireProjectModule] - Project-module resolver for adapters whose
+ *                                       SDK is the consuming project's dependency (`s3`).
+ *                                       Built by the CALLER (gna.js, from
+ *                                       `getPath('project')`) because this module is
+ *                                       framework-independent and cannot reach the path
+ *                                       globals itself; tests inject a fake SDK loader.
+ *                                       Omitted: building an `s3` driver throws (fatal).
  * @param {function} [opt.warn]        - Boot-warning sink for the per-root strategy-stamp
  *                                       checks (§4: a strategy flip on populated storage is
  *                                       unrecoverable by config, so it warns every boot; a
@@ -673,31 +1036,43 @@ function start(opt) {
         // A connector store when one was built for this driver, else the
         // embedded SQLite file inside the driver's own root — self-contained,
         // so moving the root moves its metadata with it, and never web-served
-        // because validateConfig refuses a served root.
-        var store = ( opt.stores && opt.stores[name] )
-            ? opt.stores[name]
-            : createEmbeddedMetaStore(nodePath.join(conf.root, '.meta.db'));
+        // because validateConfig refuses a served root. `s3` is STORELESS:
+        // the provider carries the metadata ON the object (immutable after
+        // upload, like the key), so there is no embedded file to create.
+        var isRemote = ( conf.adapter === 's3' );
+        var store = isRemote
+            ? null
+            : ( ( opt.stores && opt.stores[name] )
+                ? opt.stores[name]
+                : createEmbeddedMetaStore(nodePath.join(conf.root, '.meta.db')) );
 
         // Defaults, size/duration parsing and the cas hash canonicalisation —
         // ONE shared resolver, also driven by the `storage:*` CLI's offline
         // path (see resolveDriverConf).
         var resolved = resolveDriverConf(conf);
 
-        // THE strategy dispatch — validateConfig has already refused unknown
-        // and deferred names, so this lookup cannot miss on a validated boot;
-        // a direct caller bypassing validation gets a loud TypeError, which
-        // is the fail-fast it asked for. A factory throw (e.g. cas over a
-        // store lacking the refcount verbs) propagates to the caller, whose
-        // documented contract treats it as fatal.
-        built[name] = FACTORIES[conf.strategy](name, resolved, store);
+        // THE dispatch — adapter × strategy via the composition map (an
+        // adapter-less conf is a direct caller's, and behaves exactly as
+        // before: the local row). validateConfig has already refused unknown
+        // names, so a miss here is a programming error and gets its loud
+        // TypeError. A factory throw (a store lacking the refcount verbs, a
+        // missing s3 SDK) propagates to the caller, whose documented contract
+        // treats it as fatal. The 4th argument carries the injected
+        // dependencies the s3 factory needs; local factories ignore it.
+        built[name] = ADAPTER_FACTORIES[conf.adapter || 'local'][resolved.strategy](
+            name, resolved, store, { requireModule: opt.requireProjectModule }
+        );
 
         // §4 — keys are strategy-specific, so a strategy flip on populated
         // storage is unrecoverable by config edit; persist the strategy on
         // the root's own metadata and warn on mismatch, every boot, until it
         // is resolved. Fire-and-forget: with the embedded store it completes
         // synchronously, with a connector store the warning simply lands
-        // when it lands — it gates nothing.
-        checkDriverStamp(name, conf.strategy, resolved.hash || null, store, warn);
+        // when it lands — it gates nothing. NOT for s3: no store to stamp,
+        // and only one strategy to flip to.
+        if ( !isRemote ) {
+            checkDriverStamp(name, conf.strategy, resolved.hash || null, store, warn);
+        }
     }
 
     _drivers = built;
@@ -871,6 +1246,8 @@ module.exports = {
     // and factories a boot uses — in-tree consumers, versioned together.
     _resolveDriverConf       : resolveDriverConf,
     _FACTORIES                : FACTORIES,
+    _ADAPTER_FACTORIES        : ADAPTER_FACTORIES,
+    _S3_KEYS                  : S3_KEYS,
     _createEmbeddedMetaStore  : createEmbeddedMetaStore,
     _ulid                    : util.ulid,
     _parseSize               : util.parseSize,
@@ -886,5 +1263,9 @@ module.exports = {
     _DEFAULT_CAS_FSYNC        : DEFAULT_CAS_FSYNC,
     _DEFAULT_SWEEP_INTERVAL   : DEFAULT_SWEEP_INTERVAL,
     _DEFAULT_SWEEP_GRACE      : DEFAULT_SWEEP_GRACE,
+    _DEFAULT_CHUNK_SIZE       : DEFAULT_CHUNK_SIZE,
+    _DEFAULT_STREAM_FSYNC     : DEFAULT_STREAM_FSYNC,
+    _DEFAULT_SESSION_TTL      : DEFAULT_SESSION_TTL,
+    _DEFAULT_SESSION_SWEEP_INTERVAL : DEFAULT_SESSION_SWEEP_INTERVAL,
     _parseDuration            : util.parseDuration
 };

package/framework/{v0.6.7 → v0.6.8}/lib/storage/src/meta-store.js

@@ -112,8 +112,19 @@ function noop() {}
  * primitive is) and inherits the documented residual: across PROCESSES, the
  * sweep's unlink can in principle race a same-content publish in the
  * nanoseconds between claim and unlink — which is why the sweep only collects
- * blobs at zero for longer than the grace window, and why multi-process sweep
- * coordination arrives WITH the first connector implementation, not before.
+ * blobs at zero for longer than the grace window.
+ *
+ * Multi-process sweep COORDINATION was expected to arrive with the first
+ * connector store; measured against that store (couchbase, 2026-08-14), it
+ * turned out to need no election layer, because `removeIfZero`'s atomicity IS
+ * the coordination: several processes may sweep one root concurrently and
+ * exactly one claim per blob wins, every loser reading a compare-and-set
+ * failure and skipping. An election would also have made `storage:gc` report
+ * `collected: 0` on whichever process did not hold the lease. The residual
+ * above is UNCHANGED by that finding — it is put-vs-sweep, not
+ * sweeper-vs-sweeper, so no coordination between sweepers could have closed
+ * it; the grace window, the put-side heal and `storage:verify` remain its
+ * mitigations.
  *
  * @typedef  {Object} StorageMetaStore
  * @property {function(string, StorageMeta, function=): void} set    - Upsert `meta` under `key`; `fn(err, meta)`. MUST report failure through the callback, never by throwing. NEVER call it on a refcounted row — REPLACE semantics would reset the count; `acquireRef` owns those rows.