@atomicmemory/core 1.0.0

package/dist/storage/provider-metadata-projection.d.ts

@@ -0,0 +1,27 @@
+/**
+ * @file Provider-metadata projection for storage artifact rows.
+ *
+ * Raw-content adapters keep provider-native metadata in their own
+ * nested internal shape. Storage artifacts split that into two
+ * public columns: discoverable identifiers and provider details.
+ * This module is the single translation point so document-backed
+ * artifacts and direct storage artifacts expose the same
+ * allowlisted shape, and is consumed by `redactArtifactPublic` to
+ * build the public `storage_artifacts` wire response.
+ *
+ * Phase 3/4 contract: CID-shaped identifiers MUST satisfy the
+ * shared structural shape gates in `./filecoin-cid-validation.ts`
+ * before they may surface in the public output. The projector
+ * silently drops anything that fails the shape — so legacy /
+ * adversarial / manually-planted JSONB values (the Phase 4
+ * rename's pre-migration `cid` slot, sentinel strings, garbage)
+ * cannot reach the wire even on rows the SDK parser never gated.
+ * Codec-aware validation (Synapse SDK `asPieceCID`,
+ * `multiformats/cid.CID.parse`) lives at the WRITE path on the
+ * provider boundary; this module stays eager-import safe.
+ */
+export interface ProjectedArtifactProviderFields {
+    identifiers: Record<string, string>;
+    providerDetails: Record<string, unknown> | null;
+}
+export declare function projectArtifactProviderFields(provider: string, metadata: Record<string, unknown>): ProjectedArtifactProviderFields;

package/dist/storage/provider-metadata-projection.js

@@ -0,0 +1,68 @@
+/**
+ * @file Provider-metadata projection for storage artifact rows.
+ *
+ * Raw-content adapters keep provider-native metadata in their own
+ * nested internal shape. Storage artifacts split that into two
+ * public columns: discoverable identifiers and provider details.
+ * This module is the single translation point so document-backed
+ * artifacts and direct storage artifacts expose the same
+ * allowlisted shape, and is consumed by `redactArtifactPublic` to
+ * build the public `storage_artifacts` wire response.
+ *
+ * Phase 3/4 contract: CID-shaped identifiers MUST satisfy the
+ * shared structural shape gates in `./filecoin-cid-validation.ts`
+ * before they may surface in the public output. The projector
+ * silently drops anything that fails the shape — so legacy /
+ * adversarial / manually-planted JSONB values (the Phase 4
+ * rename's pre-migration `cid` slot, sentinel strings, garbage)
+ * cannot reach the wire even on rows the SDK parser never gated.
+ * Codec-aware validation (Synapse SDK `asPieceCID`,
+ * `multiformats/cid.CID.parse`) lives at the WRITE path on the
+ * provider boundary; this module stays eager-import safe.
+ */
+import { isIpfsCid, isPieceCid } from './filecoin-cid-validation.js';
+export function projectArtifactProviderFields(provider, metadata) {
+    if (provider === 'filecoin')
+        return projectFilecoin(metadata);
+    return { identifiers: {}, providerDetails: null };
+}
+function projectFilecoin(metadata) {
+    const filecoin = nestedRecord(metadata, 'filecoin');
+    if (filecoin === null)
+        return { identifiers: {}, providerDetails: null };
+    const identifiers = {};
+    // Phase 4: internal sidecar key is `ipfs_cid` (the legacy
+    // ambiguous `cid` slot is silently dropped here AND rejected by
+    // the public Zod schema). `car_root_cid` shares the broader
+    // CIDv1 shape gate — CAR roots are CIDv1 strings.
+    const ipfsCid = filecoin['ipfs_cid'];
+    if (isIpfsCid(ipfsCid))
+        identifiers.ipfsCid = ipfsCid;
+    const pieceCid = filecoin['piece_cid'];
+    if (isPieceCid(pieceCid))
+        identifiers.pieceCid = pieceCid;
+    const carRootCid = filecoin['car_root_cid'];
+    if (isIpfsCid(carRootCid))
+        identifiers.carRootCid = carRootCid;
+    // `data_set_id` is intentionally exposed as a non-empty string
+    // — the artifact-identifier wire surfaces the per-row Synapse
+    // data-set scalar for operators that need to correlate with
+    // their own provider tooling. No CID shape applies.
+    const dataSetId = filecoin['data_set_id'];
+    if (typeof dataSetId === 'string' && dataSetId.length > 0) {
+        identifiers.dataSetId = dataSetId;
+    }
+    return {
+        identifiers,
+        // No public providerDetails for filecoin until the provider
+        // has explicitly projected Synapse-shaped public fields. The
+        // public wire carries CIDs (plus `dataSetId`) only.
+        providerDetails: null,
+    };
+}
+function nestedRecord(source, key) {
+    const value = source[key];
+    if (value === null || typeof value !== 'object' || Array.isArray(value))
+        return null;
+    return value;
+}

package/dist/storage/providers/filecoin/backend.d.ts

@@ -0,0 +1,42 @@
+/**
+ * @file `RawContentStore` adapter wrapping a `FilecoinProviderClient`.
+ *
+ * Two reach-paths to keep in mind:
+ *
+ *   - Direct managed uploads (`POST /v1/storage/artifacts?mode=managed`)
+ *     are rejected upstream by the
+ *     `DIRECT_MANAGED_UNSUPPORTED_PROVIDERS` carve-out in
+ *     `services/storage-service.ts`, so the route returns a typed
+ *     501 and no `storage_artifacts` row is ever created — this
+ *     adapter is not invoked on that path.
+ *   - Document-path uploads against `RAW_STORAGE_PROVIDER=filecoin`
+ *     construct this adapter (via `storage/factory.ts`) and call
+ *     `put`/`get`/`head`/`delete` here. The adapter delegates to
+ *     the underlying `FilecoinProviderClient`.
+ *
+ * Status mapping — Filecoin is `retrievalConsistency: 'eventual'`,
+ * so `put` ALWAYS reports `'pending'` regardless of
+ * `UploadResult.complete`. The reconciler / `head` path is
+ * responsible for promoting `blob_pending → blob_available` after
+ * a real retrievability check.
+ *
+ * Capabilities advertised:
+ *
+ *   - `addressing: 'content'`            (PieceCID-addressed)
+ *   - `retrievalConsistency: 'eventual'` (deal lifecycle is async)
+ *   - `deleteSemantics: 'tombstone'`     (provider tombstone only)
+ *   - `supportsHead: true`               (via context.pieceStatus)
+ *   - `supportsGet: true`                (via storage.download)
+ */
+import type { FilecoinProviderClient } from './provider-client.js';
+import type { PutRawContentInput, RawContentDeleteResult, RawContentGetResult, RawContentHeadResult, RawContentHints, RawContentStore, RawContentStoreCapabilities, StoredRawContent } from '../../raw-content-store.js';
+export declare class FilecoinRawContentStore implements RawContentStore {
+    private readonly client;
+    readonly provider = "filecoin";
+    readonly capabilities: RawContentStoreCapabilities;
+    constructor(client: FilecoinProviderClient);
+    put(input: PutRawContentInput): Promise<StoredRawContent>;
+    get(storageUri: string): Promise<RawContentGetResult>;
+    head(storageUri: string, hints?: RawContentHints): Promise<RawContentHeadResult>;
+    delete(storageUri: string, hints?: RawContentHints): Promise<RawContentDeleteResult>;
+}

package/dist/storage/providers/filecoin/backend.js

@@ -0,0 +1,250 @@
+/**
+ * @file `RawContentStore` adapter wrapping a `FilecoinProviderClient`.
+ *
+ * Two reach-paths to keep in mind:
+ *
+ *   - Direct managed uploads (`POST /v1/storage/artifacts?mode=managed`)
+ *     are rejected upstream by the
+ *     `DIRECT_MANAGED_UNSUPPORTED_PROVIDERS` carve-out in
+ *     `services/storage-service.ts`, so the route returns a typed
+ *     501 and no `storage_artifacts` row is ever created — this
+ *     adapter is not invoked on that path.
+ *   - Document-path uploads against `RAW_STORAGE_PROVIDER=filecoin`
+ *     construct this adapter (via `storage/factory.ts`) and call
+ *     `put`/`get`/`head`/`delete` here. The adapter delegates to
+ *     the underlying `FilecoinProviderClient`.
+ *
+ * Status mapping — Filecoin is `retrievalConsistency: 'eventual'`,
+ * so `put` ALWAYS reports `'pending'` regardless of
+ * `UploadResult.complete`. The reconciler / `head` path is
+ * responsible for promoting `blob_pending → blob_available` after
+ * a real retrievability check.
+ *
+ * Capabilities advertised:
+ *
+ *   - `addressing: 'content'`            (PieceCID-addressed)
+ *   - `retrievalConsistency: 'eventual'` (deal lifecycle is async)
+ *   - `deleteSemantics: 'tombstone'`     (provider tombstone only)
+ *   - `supportsHead: true`               (via context.pieceStatus)
+ *   - `supportsGet: true`                (via storage.download)
+ */
+import { createHash } from 'node:crypto';
+import { FilecoinProviderError } from './errors.js';
+import { readFilecoinDataSetIdHint, readFilecoinDeleteHints } from './hints.js';
+import { requireIpfsCid } from './ipfs-cid.js';
+import { requirePieceCid } from './piece-cid.js';
+import { formatPieceUri, parsePieceUri } from './uri.js';
+const FILECOIN_CAPABILITIES = {
+    addressing: 'content',
+    retrievalConsistency: 'eventual',
+    deleteSemantics: 'tombstone',
+    supportsHead: true,
+    supportsGet: true,
+};
+export class FilecoinRawContentStore {
+    client;
+    provider = 'filecoin';
+    capabilities = FILECOIN_CAPABILITIES;
+    constructor(client) {
+        this.client = client;
+    }
+    async put(input) {
+        const result = await this.client.put({
+            key: input.key,
+            body: input.body,
+            ...(input.contentType ? { contentType: input.contentType } : {}),
+        });
+        // Thread the boundary's `driver` into the sidecar — never
+        // hardcode `'synapse'` here. Production: the live Synapse
+        // client → `'synapse'`. Tests / a future filecoin-pin client
+        // → that driver's literal. Hardcoding would lie for the
+        // second driver and undermine the driver-agnostic invariant.
+        return toStoredRawContent(input, result, this.client.driver);
+    }
+    async get(storageUri) {
+        const result = await this.client.get({ storageUri });
+        return {
+            body: result.body,
+            metadata: {
+                contentLength: result.body.length,
+                contentType: null,
+                contentHash: null,
+                providerMetadata: result.providerMetadata,
+            },
+        };
+    }
+    async head(storageUri, hints) {
+        const dataSetId = readFilecoinDataSetIdHint(hints);
+        const result = await this.client.head({
+            storageUri,
+            ...(dataSetId !== null ? { dataSetId } : {}),
+        });
+        // Filecoin lifecycle gate: a Synapse piece is RETRIEVABLE only
+        // after PDP proof has landed. The reconciler treats any
+        // `exists=true` head as "promote blob_pending → blob_available",
+        // so we MUST NOT report exists=true on an unproven piece —
+        // otherwise the row would promote before the bytes are actually
+        // retrievable. An unproven-but-known piece is reported as
+        // `exists=false` (without a permanent-failure marker) so the
+        // reconciler keeps the row pending and retries on a later tick.
+        if (!result.exists || !result.proven) {
+            return { exists: false, metadata: null };
+        }
+        return {
+            exists: true,
+            metadata: {
+                contentLength: 0,
+                contentType: null,
+                contentHash: null,
+                providerMetadata: result.providerMetadata,
+            },
+        };
+    }
+    async delete(storageUri, hints) {
+        // Extract BOTH the data-set-id (for context resolution) and
+        // the per-copy piece-id (for direct deletePiece-by-id) from
+        // the sidecar. The cleanup loop plumbs `raw_storage_metadata`
+        // through `ManagedBlobRef` so production cleanup carries the
+        // same hints the reconciler uses on `head`. CID-based delete
+        // remains the fallback when hints are missing or malformed —
+        // useful for legacy rows that pre-date the sidecar — but
+        // Synapse cannot resolve a freshly-uploaded piece by CID
+        // before PDP proof lands, so `pieceId` is required for the
+        // most common case (delete shortly after upload).
+        const { dataSetId, pieceId } = readFilecoinDeleteHints(hints);
+        const result = await this.client.delete({
+            storageUri,
+            ...(dataSetId !== null ? { dataSetId } : {}),
+            ...(pieceId !== null ? { pieceId } : {}),
+        });
+        // Phase 7 — pass `result.txHash` through as the internal-only
+        // billing/cost-impact metadata. `RawContentDeleteResult.txHash`
+        // is documented as INTERNAL: it MUST NOT cross any public
+        // route boundary, and `cleanupManagedBlobs`'s closed-key
+        // success DTO never includes it. The legitimate downstream
+        // consumer is the `filecoin.delete.tombstoned` observability
+        // emitter (operator-side telemetry only).
+        return {
+            deleted: result.deleted,
+            semantics: 'tombstoned',
+            ...(result.txHash !== undefined ? { txHash: result.txHash } : {}),
+        };
+    }
+}
+/**
+ * Translate a Filecoin upload result to the generic
+ * `StoredRawContent` shape persisted by the upload service. The
+ * `content_hash` is the SHA-256 of the PLAINTEXT bytes the caller
+ * passed in (the codec/AES-GCM layer already wrapped them on the
+ * way down). Synapse reports the on-wire byte count in
+ * `result.sizeBytes`; we persist `input.body.length` so the row's
+ * `size_bytes` matches the plaintext contract the rest of the
+ * pipeline uses.
+ *
+ * Status mapping — Filecoin is `retrievalConsistency: 'eventual'`,
+ * so `put` ALWAYS reports `'pending'` regardless of
+ * `UploadResult.complete`. A Synapse-side `complete=true` means the
+ * SDK saw the requested copies stored on SPs; it does NOT prove the
+ * data set's proof has landed or that the bytes are retrievable
+ * via `download()`. The reconciler / `head` path is responsible for
+ * promoting `blob_pending → blob_available` after a real
+ * retrievability check. The original `complete` flag and copy
+ * states are preserved in the internal
+ * `raw_storage_metadata.filecoin` sidecar so the reconciler can
+ * make decisions without re-querying the provider.
+ */
+function toStoredRawContent(input, result, driver) {
+    // Phase 3 defense-in-depth: validate AND canonicalize before
+    // any identifier reaches `storage_artifacts.uri` or the
+    // `raw_storage_metadata.filecoin.piece_cid` sidecar. Both
+    // halves of `result` are parsed through the live Synapse SDK
+    // (`@filoz/synapse-core/piece.asPieceCID`) via
+    // `requirePieceCid`, which ALSO returns the SDK's canonical
+    // base32-lower CIDv1 multibase string. `asPieceCID` accepts
+    // any parser-valid multibase encoding of the same PieceCID
+    // (e.g. base58btc `z…`); the canonicalization step here
+    // collapses those to the canonical `bafk…` form before
+    // persistence so a future driver returning a non-canonical
+    // multibase variant cannot leave a non-canonical URI or
+    // sidecar value behind. The cross-field mismatch check then
+    // compares the canonical forms — catching the case where
+    // both halves parse but refer to different identifiers.
+    const canonicalPieceCid = requirePieceCid(result.pieceCid, 'upload result');
+    const canonicalFromUri = parsePieceUri(result.storageUri);
+    if (canonicalFromUri !== canonicalPieceCid) {
+        throw new FilecoinProviderError('identifier_mismatch', 'Provider client returned a storage URI whose PieceCID disagrees with the result.pieceCid field.');
+    }
+    // Phase 4: validate AND canonicalize the optional IPFS/CAR-root
+    // CID using a real `multiformats/cid` parse. The slot is opt-in
+    // (live Synapse leaves it `undefined`); when populated, the
+    // sidecar carries the canonical CIDv1 string. The canonical
+    // storage URI stays PieceCID-based — `ipfsCid` does not change
+    // row identity.
+    const canonicalIpfsCid = result.ipfsCid === undefined
+        ? undefined
+        : requireIpfsCid(result.ipfsCid, 'upload result ipfsCid');
+    return {
+        storageUri: formatPieceUri(canonicalPieceCid),
+        storageProvider: 'filecoin',
+        contentHash: sha256Hex(input.body),
+        sizeBytes: input.body.length,
+        status: 'pending',
+        providerMetadata: toProviderMetadata(result, driver, canonicalPieceCid, canonicalIpfsCid),
+    };
+}
+function sha256Hex(buf) {
+    return createHash('sha256').update(buf).digest('hex');
+}
+/**
+ * Build the internal-only `raw_storage_metadata.filecoin` sidecar
+ * the document-side pipeline writes. The PUBLIC projection runs
+ * later through `src/storage/filecoin-public-metadata.ts`, which
+ * flattens `copies[]` into `copy_count` / `provider_ids` /
+ * `copy_statuses` and drops everything else.
+ *
+ * Per-copy `status` is `'accepted'` on upload: the Synapse SDK's
+ * `UploadResult.copies[]` only proves the SP accepted the bytes
+ * (PDP proof has not landed yet, retrieval is unconfirmed). The
+ * reconciler / `head` path advances copies to `'available'` once
+ * `StorageContext.pieceStatus` reports proof + retrieval.
+ * `complete` carries the raw `UploadResult.complete` flag so the
+ * reconciler can distinguish "all copies stored" from "partial".
+ */
+function toProviderMetadata(result, driver, canonicalPieceCid, canonicalIpfsCid) {
+    return {
+        filecoin: {
+            driver,
+            // Persist the SDK's canonical multibase form (not
+            // `result.pieceCid`) so a non-canonical-but-parser-valid
+            // input from a misbehaving driver still lands on disk as
+            // the canonical `bafk…` string.
+            piece_cid: canonicalPieceCid,
+            // Phase 4: optional IPFS / CAR-root identity. Snake_case
+            // for the JSONB key; only present when the driver
+            // populated `result.ipfsCid`. Omitted entirely (vs. set to
+            // `null`) so closed-key assertions over the sidecar stay
+            // stable for the live Synapse driver, which never writes
+            // this slot today.
+            ...(canonicalIpfsCid === undefined ? {} : { ipfs_cid: canonicalIpfsCid }),
+            data_set_id: result.copies[0]?.dataSetId,
+            copies: result.copies.map((copy) => ({
+                provider_id: copy.providerId,
+                data_set_id: copy.dataSetId,
+                // Omit `piece_id` entirely when the driver reported a
+                // sentinel/non-positive value. Writing `'0'` (or any
+                // non-positive decimal) would later trip the hint
+                // reader's positive-decimal-bigint validator and emit a
+                // spurious `filecoin.hint.malformed` diagnostic on every
+                // delete. Absent silently falls back to the CID-based
+                // delete lookup, which is the intended behaviour when a
+                // piece-id hint is unavailable.
+                ...(copy.pieceId !== undefined ? { piece_id: copy.pieceId } : {}),
+                role: copy.role,
+                status: 'accepted',
+            })),
+            failed_attempts: result.failedAttempts.length,
+            requested_copies: result.requestedCopies,
+            complete: result.complete,
+        },
+    };
+}

package/dist/storage/providers/filecoin/config.d.ts

@@ -0,0 +1,70 @@
+/**
+ * @file Filecoin provider configuration.
+ *
+ * `parseFilecoinProviderConfig(env)` is the only env reader inside
+ * `src/storage/providers/filecoin/*`. Central `src/config.ts`
+ * collects `process.env` into a plain object and hands it to this
+ * function; the provider module never reaches for `process.env`
+ * itself, matching the workspace rule against direct env access in
+ * business code.
+ *
+ * The cross-provider guard — rejecting any `RAW_STORAGE_FILECOIN_*`
+ * environment variable when `RAW_STORAGE_PROVIDER` is non-filecoin —
+ * lives in central `src/config.ts`. Reason:
+ * `parseFilecoinProviderConfig` is only invoked when the provider IS
+ * filecoin, so the cross-provider rejection has to fire at the
+ * central seam to catch the misconfiguration. This module exports
+ * `collectFilecoinProviderEnvKeys` so the central code can compute
+ * which RAW_STORAGE_FILECOIN_* keys an operator set without
+ * embedding the prefix string in two places.
+ *
+ * v1 credential model: operator-owned deployment credentials.
+ * Storage-profile / per-org / per-user credential ownership is
+ * deferred to a future release. `RAW_STORAGE_FILECOIN_PRIVATE_KEY` is
+ * validated syntactically (`^0x[a-fA-F0-9]{64}$`) at startup. The
+ * value is NEVER logged on rejection; no account derivation, no
+ * chain, balance, and provider checks are handled by the readiness
+ * probes in `./readiness.ts`.
+ */
+export type FilecoinNetwork = 'calibration' | 'mainnet';
+export type FilecoinDriver = 'synapse' | 'filecoin_pin';
+/**
+ * Parsed deployment-credential block. Constructed by
+ * `parseFilecoinProviderConfig` at startup; held on the
+ * runtime config and consumed by the Synapse client factory.
+ */
+export interface FilecoinProviderConfig {
+    readonly driver: FilecoinDriver;
+    readonly network: FilecoinNetwork;
+    /** 0x-prefixed 32-byte hex string. Never logged on rejection. */
+    readonly privateKey: string;
+    readonly source: string;
+    readonly withCdn: boolean;
+    readonly providerIds: ReadonlyArray<string>;
+    readonly copies: number | null;
+    readonly dataSetMetadata: Readonly<Record<string, string | number | boolean>>;
+    readonly maxUploadBytes: number | null;
+    readonly minUploadBytes: number | null;
+    readonly uploadTimeoutMs: number | null;
+    readonly retrievalTimeoutMs: number | null;
+}
+/**
+ * Syntactic-only validator for `RAW_STORAGE_FILECOIN_PRIVATE_KEY`.
+ * Accepts a 32-byte hex string with `0x` prefix (case-insensitive).
+ * Throws on any other shape WITHOUT echoing the value — the message
+ * names only the env var and the expected pattern.
+ */
+export declare function parseFilecoinPrivateKey(value: string): string;
+/**
+ * Return the names of every `RAW_STORAGE_FILECOIN_*` key the
+ * operator set to a non-empty value. Used by the central
+ * cross-provider guard in `src/config.ts` so non-filecoin
+ * deployments cannot accidentally carry Filecoin config.
+ */
+export declare function collectFilecoinProviderEnvKeys(env: Record<string, string | undefined>): ReadonlyArray<string>;
+/**
+ * Parse + validate the Synapse-shaped Filecoin provider env block.
+ * Only invoked when `RAW_STORAGE_PROVIDER=filecoin`; the cross-
+ * provider guard runs in `src/config.ts` before this function.
+ */
+export declare function parseFilecoinProviderConfig(env: Record<string, string | undefined>): FilecoinProviderConfig;