@ocfl/ocfl 0.3.0 → 1.0.0-beta.1

package/src/utils.ts

@@ -0,0 +1,94 @@
+import { OcflConstants } from './constants.ts';
+import type { OcflStore } from './store.ts';
+import type { OcflVersion } from './types.ts';
+
+const { OCFL_VERSIONS, NAMASTE_T } = OcflConstants;
+
+function isErrnoException(error: unknown): error is NodeJS.ErrnoException {
+  return error instanceof Error && 'code' in error;
+}
+
+/**
+ * Run `asyncFn` over every element of `inputStack` with at most `count` calls in flight.
+ * Rejections are returned in the result array rather than thrown.
+ * @param inputStack - Consumed during the process.
+ * @param count - Maximum number of concurrency
+ */
+async function parallelize<T, R>(inputStack: T[], asyncFn: (input: T) => Promise<R>, count = 10): Promise<(R | Error)[]> {
+  const promises: Promise<void>[] = [];
+  let len = Math.min(inputStack.length, count);
+  const result: (R | Error)[] = [];
+  while (len--) {
+    promises.push(
+      (async () => {
+        while (inputStack.length) {
+          const input = inputStack.pop() as T;
+          const index = inputStack.length;
+          try {
+            result[index] = await asyncFn(input);
+          } catch (error) {
+            result[index] = error instanceof Error ? error : new Error(String(error), { cause: error });
+          }
+        }
+      })().catch(() => {}),
+    );
+  }
+  await Promise.all(promises);
+  return result;
+}
+
+async function isDirEmpty(store: OcflStore, dirPath: string): Promise<boolean> {
+  let de: unknown;
+  try {
+    const dir = await store.opendir(dirPath);
+    de = await dir.read();
+    await dir.close();
+  } catch (error) {
+    return isErrnoException(error) && error.code === 'ENOENT';
+  }
+  return !de;
+}
+
+/**
+ * Check if there is any valid namaste in the rootPath and return the version number.
+ * @returns The version, `''` if a namaste exists but is invalid, or `undefined` if there is none.
+ */
+async function findNamasteVersion(store: OcflStore, prefix: string, rootPath: string): Promise<OcflVersion | '' | undefined> {
+  const namastePath = `${rootPath}/${NAMASTE_T}${prefix}`;
+  let version: OcflVersion | null | undefined;
+  try {
+    version = await Promise.any(OCFL_VERSIONS.map(async (v) => ((await store.readFile(namastePath + v, 'utf8')) === `${prefix}${v}\n` ? v : null)));
+    if (!version) {
+      return '';
+    }
+  } catch (error) {
+    if (!(error instanceof AggregateError) || error.errors.some((e) => !isErrnoException(e) || e.code !== 'ENOENT')) {
+      throw error;
+    }
+  }
+  return version ?? undefined;
+}
+
+type TypedArray = ArrayLike<number> & { buffer: ArrayBufferLike; byteLength: number; set(array: ArrayLike<number>, offset?: number): void };
+
+function isTypedArray(obj: unknown): obj is TypedArray {
+  return ArrayBuffer.isView(obj) && !(obj instanceof DataView);
+}
+
+function joinTypedArray(target: ArrayBuffer | TypedArray, source: ArrayBuffer | TypedArray): TypedArray {
+  const sourceArray = isTypedArray(source) ? source : new Uint8Array(source);
+  const [targetBuffer, targetArray] = isTypedArray(target) ? [target.buffer as ArrayBuffer, target] : [target, new Uint8Array(target)];
+  const targetLength = targetArray.byteLength;
+  const totalSize = targetLength + sourceArray.byteLength;
+  let mergedArray = targetArray;
+  if (targetBuffer.resizable && targetBuffer.maxByteLength >= totalSize) {
+    targetBuffer.resize(totalSize);
+  } else {
+    mergedArray = new Uint8Array(totalSize);
+    mergedArray.set(targetArray, 0);
+  }
+  mergedArray.set(sourceArray, targetLength);
+  return mergedArray;
+}
+
+export { findNamasteVersion, isDirEmpty, isErrnoException, joinTypedArray, parallelize };

package/src/validation.ts

@@ -0,0 +1,46 @@
+// Messages for the OCFL 1.1 validation codes this library raises: https://ocfl.io/1.1/spec/validation-codes.html
+const ERRORS = {
+  E007: 'The text contents of the [version declaration] file must be the same as dvalue, followed by a newline (\\n).',
+  E036: 'An OCFL Object Inventory MUST include the following keys: [id, type, digestAlgorithm, head].',
+  E040: '[head] MUST be the version directory name with the highest version number.',
+  E050: 'The keys of [the state JSON object] are digest values, each of which MUST correspond to an entry in the manifest of the inventory.',
+  E060: 'The digest sidecar file must contain the digest of the inventory file.',
+  E069: 'An OCFL Storage Root MUST contain a Root Conformance Declaration identifying it as such.',
+  E080: 'The text contents of [the OCFL version declaration file] MUST be the same as dvalue, followed by a newline (\\n).',
+  E082: 'OCFL Object Roots MUST be stored either as the terminal resource at the end of a directory storage hierarchy or as direct children of a containing OCFL Storage Root.',
+};
+
+type ErrorCode = keyof typeof ERRORS;
+
+interface ValidationErrorOptions {
+  code: ErrorCode;
+  description: string;
+  path: string;
+  detail?: string | undefined;
+}
+
+class ValidationError extends Error {
+  declare code: ErrorCode;
+  declare description: string;
+  declare path: string;
+  declare detail?: string | undefined;
+
+  constructor(options: ValidationErrorOptions) {
+    super(`${options.code}: ${options.description} At ${options.path}${options.detail ? ` (${options.detail})` : ''}`);
+    this.name = 'ValidationError';
+    Object.assign(this, options);
+  }
+}
+
+/**
+ * @param code - OCFL validation code, eg 'E060'
+ * @param path - Path the problem was found at
+ * @param detail - Extra context appended to the message
+ */
+export const createError = (code: ErrorCode, path: string, detail?: string) => {
+  const description = Object.hasOwn(ERRORS, code) ? ERRORS[code] : undefined;
+  if (!description) {
+    throw new RangeError(`Unknown OCFL validation code ${code}`);
+  }
+  return new ValidationError({ code, description, path, detail });
+};

package/index.js

@@ -1,129 +0,0 @@
-/**
- * @module ocfl
- */
-
-const { OcflConstants, OcflDigest, OcflExtension, OcflStorageLayout, OcflStore,
-  OcflObject, OcflObjectInventory, OcflObjectTransactionImpl,
-  OcflStorage, OcflStorageImpl,
-  extensions
-} = require('./lib/index.js');
-
-/**
- * @typedef {Object} StorageLayout
- * @property {typeof extensions.FlatDirectStorageLayout} FlatDirectStorageLayout
- * @property {typeof extensions.FlatOmitPrefixStorageLayout} FlatOmitPrefixStorageLayout
- * @property {typeof extensions.HashAndIdNTupleStorageLayout} HashAndIdNTupleStorageLayout
- * @property {typeof extensions.HashedNTupleStorageLayout} HashedNTupleStorageLayout
- * @property {typeof extensions.NTupleOmitPrefixStorageLayout} NTupleOmitPrefixStorageLayout
- * @property {typeof extensions.PathDirectStorageLayout} PathDirectStorageLayout
- */
-
-/** @typedef {import('./lib/types').OcflExtensionConfig} OcflExtensionConfig */
-/** @typedef {import('./lib/types').OcflObjectConfig} OcflObjectConfig */
-/** @typedef {import('./lib/types').OcflStorageConfig} OcflStorageConfig */
-
-/**
- * Create a storage layout from extension name or config
- * @param {string|OcflExtensionConfig} config - Layout name or config
- */
-function layout(config) {
-  let name, c;
-  if (typeof config === 'string') {
-    name = config;
-  } else {
-    name = config.extensionName;
-    c = config;
-  }
-  return OcflStorageLayout.class(name).create(c);
-}
-let storageLayout = Object.assign(layout, /** @type {StorageLayout} */(OcflStorageLayout.layout))
-
-/**
- * Creates a new OCFL core API with a specific backend store.
- * Any backend store implementation module must return the result of this method. 
- * @param {typeof OcflStore} store 
- * @param {SO} defaultOptions
- * @template SO
- * @returns Top level OCFL API interface that provides storage and object creation.
- */
-function implementOcfl(store, defaultOptions) {
-  return {
-    OCFL_VERSIONS: OcflConstants.OCFL_VERSIONS,
-
-    /** Built-in storage layout classes */
-    StorageLayout: OcflStorageLayout.layout,
-
-    /**
-     * Instantiate a storage layout from extension name or config
-     * @param {string | OcflExtensionConfig} config - Layout name or config
-     */
-    storageLayout(config) {
-      let name, c;
-      if (typeof config === 'string') {
-        name = config;
-      } else {
-        name = config.extensionName;
-        c = config;
-      }
-      let cls = OcflStorageLayout.class(name)
-      if (!cls) throw new Error(`Layout "${name}" is not supported.`);
-      return cls.create(c);
-    },
-
-    /**
-     * Instantiate an OCFL Object
-     * @param {OcflObjectConfig} config
-     * @param {SO} [storeOptions]
-     */
-    object(config, storeOptions = defaultOptions) {
-      return new OcflObject(config, store.getInstance(storeOptions));
-    },
-
-    /**
-     * Instantiate an OCFL Storage root
-     * @param {OcflStorageConfig} config
-     * @param {SO} [storeOptions]
-     * @return {OcflStorage}
-     */
-    storage(config, storeOptions = defaultOptions) {
-      return new OcflStorageImpl(config, store.getInstance(storeOptions));
-    },
-
-    /**
-     * Create a new OCFL Storage
-     * @param {OcflStorageConfig} config
-     * @param {SO} [storeOptions]
-     * @return {Promise<OcflStorage>}
-     */
-    async createStorage(config, storeOptions = defaultOptions) {
-      return (new OcflStorageImpl(config, store.getInstance(storeOptions))).create();
-    },
-
-    /**
-     * Load an existing OCFL Storage
-     * @param {OcflStorageConfig} config
-     * @param {SO} [storeOptions]
-     * @return {Promise<OcflStorage>}
-     */
-    async loadStorage(config, storeOptions = defaultOptions) {
-      return (new OcflStorageImpl(config, store.getInstance(storeOptions))).load();
-    }
-  }
-}
-
-module.exports = {
-  OcflConstants,
-  OcflDigest,
-  OcflExtension, OcflStorageLayout,
-  OcflStore,
-  OcflObject,
-  OcflObjectInventory,
-  OcflObjectTransactionImpl,
-  OcflStorage, OcflStorageImpl,
-  implementOcfl,
-  extensions
-};
-
-for (let name in extensions) {
-  extensions[name].register();
-}

package/index.mjs

@@ -1,18 +0,0 @@
-//@ts-check
-// export * from './lib/storage.js'; 
-// export * from './lib/object.js'; 
-// export * from './lib/digest.js'; 
-//export * from './lib/extension.js'; 
-//export * from './lib/constants.js'; 
-
-export * from './index.js'; 
-import * as ocfl from './index.js';
-export default ocfl;
-//export const { DIGEST, DIGEST_FIXITY, INVENTORY_NAME, NAMASTE_PREFIX_OBJECT, NAMASTE_PREFIX_STORAGE, OCFL_VERSION } = ocfl
-// export const { OcflObject, OcflObjectInventory, OcflObjectTransaction, OcflObjectTransactionImpl,
-//   OcflStorage, OcflStorageImpl,
-//   createProxy, digestFrom, digestFromStream, 
-//   DIGEST, DIGEST_FIXITY, INVENTORY_NAME, NAMASTE_PREFIX_OBJECT, NAMASTE_PREFIX_STORAGE, OCFL_VERSION } = ocfl;
-//export const {} = ocfl;
-//export { default as ocfl } from './index.js';
-//export {Storage, Object} from './index.js';

package/lib/constants.js

@@ -1,24 +0,0 @@
-/**
- * @module
- * default constants 
- */
-// module.exports = {
-//   OCFL_VERSION: 1.1,
-//   NAMASTE_PREFIX_OBJECT: '0=ocfl_object_',
-//   NAMASTE_PREFIX_STORAGE: '0=ocfl_',
-//   INVENTORY_NAME: 'inventory.json'
-// };
-const OcflConstants = {
-  OCFL_VERSION: '1.1',
-  OCFL_VERSIONS: Object.freeze(['1.1', '1.0']),
-  OCFL_LAYOUT: 'ocfl_layout.json',
-  NAMASTE_T: '0=',
-  NAMASTE_PREFIX_OBJECT: 'ocfl_object_',
-  NAMASTE_PREFIX_STORAGE: 'ocfl_',
-  INVENTORY_NAME: 'inventory.json',
-  EXTENSIONS_DIR: 'extensions',
-  EXTENSION_CONFIG: 'config.json',
-  DIGEST_ALGORITHM: 'sha512'
-};
-
-module.exports = { OcflConstants };

package/lib/digest.js

@@ -1,307 +0,0 @@
-//@ts-check
-
-const hashWasm = require('hash-wasm');
-const { sha512_256 } = require('js-sha512'); // fallback for sha512/256 algorithm, which is not included in hash-wasm
-//const worker = require('node:worker_threads');
-const { enumeration } = require('./enum.js');
-const { testSymbol } = require('./utils.js');
-
-const CONTENT = enumeration(['sha256', 'sha512']);
-const FIXITY = enumeration(['sha256', 'sha512', 'md5', 'sha1', 'blake2b-512', 'blake2b-160', 'blake2b-256', 'blake2b-384', 'sha512/256', 'size', 'crc32']);
-
-/**
- * @typedef {((outputType: "binary") => Uint8Array) & ((outputType?: "hex") => string)} IDigest
- * @typedef {((outputType: "binary") => {[key:string]: Uint8Array}) & ((outputType?: "hex") => {[key:string]: string})} IMultiDigest
- * @typedef {WritableStream & { update(data: IDataType): void, digest: IDigest }} WritableStreamHasher 
- * @typedef {WritableStream & { update(data: IDataType): void, digest: IMultiDigest }} WritableStreamMultiHasher 
- */
-
-/** 
- * @type { Object.<string, CommonHasher[]> }
- */
-const hasherCache = {};
-/** 
- * @type { Object.<string, CommonHasher> }
- */
-const hasherSyncCache = {};
-// for (let algo of FIXITY) {
-//   algorithms[algo] = function () { return crypto.createHash(algo); }
-// }
-
-/** @type { WeakMap<CommonHasher, boolean>} */
-const hasherPending = new WeakMap();
-
-/** @type { {[key: string]: () => Promise<CommonHasher>} } */
-const hasherFactory = {};
-
-/**
- * Register a hash algorithm and it's factory function.
- * @param {string} name 
- * @param {() => Promise<CommonHasher>} factory 
- */
-function registerHasherFactory(name, factory) {
-  if (typeof factory === 'function') {
-    hasherFactory[name] = factory;
-  }
-}
-
-function hasAlgorithm(name) {
-  return name in hasherFactory;
-}
-
-(function init() {
-  for (const algo of FIXITY) {
-    if (algo.startsWith('blake2b')) {
-      const bits = parseInt(algo.split('-')[1]);
-      registerHasherFactory(algo, async () => hashWasm.createBLAKE2b(bits));
-    } else if (algo === 'sha512/256') {
-      registerHasherFactory(algo, async () => {
-        let h = sha512_256.create();
-        /** @type {CommonHasher} */
-        const hasher = {
-          digestSize: 32,
-          init() { h = sha512_256.create(); return this; },
-          update(data) { h.update(/** @type {any} */(data)); return this },
-          // @ts-ignore
-          digest(encoding) {
-            if (encoding === 'binary') return new Uint8Array(h.digest());
-            else return h.hex();
-          }
-        };
-        return hasher;
-      });
-    } else if (algo === 'size') {
-      registerHasherFactory(algo, async () => {
-        let encoder;
-        let size = 0;
-        /** @type {CommonHasher} */
-        const hasher = {
-          digestSize: 1,
-          init() {
-            size = 0;
-            encoder = new TextEncoder();
-            return this;
-          },
-          update(data) {
-            if (data) {
-              if (typeof data === 'string') size += encoder.encode(data).length;
-              else size += data.byteLength;
-            }
-            return this;
-          },
-          // @ts-ignore
-          digest(encoding) {
-            if (encoding === 'binary') return new Uint8Array(new BigUint64Array([BigInt(size)]).buffer);
-            else return '' + size;
-          }
-        };
-        return hasher;
-      });
-    } else {
-      registerHasherFactory(algo, hashWasm['create' + algo.toUpperCase()]);
-    }
-  }
-})();
-// (async () => {
-//   for (const name in hasherFactory) {
-//     const factory = hasherFactory[name];
-//     const h = await factory();
-//     console.log(name);
-//     console.log(h);
-//     console.log(h.init().update('test').digest().length)
-//   }
-// })();
-
-/**
- * 
- * @param {*} hasher
- * @returns {hasher is MultiHasher}
- */
-function isMulti(hasher) {
-  return hasher.multi;
-}
-
-/**
- * Create a hasher instance for the given algorithm or get it from cache if available.
- * @param {string} algorithm 
- */
-async function createHasher(algorithm) {
-  const cacheArray = hasherCache[algorithm];
-  let hash = (cacheArray ?? []).find(hasher => !hasherPending.get(hasher));
-  if (!hash) {
-    const factory = hasherFactory[algorithm];
-    if (!factory) throw new Error(`Unsupported digest algorithm: ${algorithm}`);
-    hash = await factory();
-    if (cacheArray) cacheArray.push(hash);
-    else hasherCache[algorithm] = [hash];
-  }
-  hasherPending.set(hash, true);
-  hash.init();
-  return hash;
-}
-
-/**
- * Create a hasher instance for calculating multiple types of digest in one go.
- * @param {string|string[]} algorithms 
- */
-async function createMultiHasher(algorithms) {
-  const algoNames = Array.isArray(algorithms) ? (algorithms.length ? [...(new Set(algorithms))] : ['sha512']) : [algorithms];
-  const hashers = await Promise.all(algoNames.map(createHasher));
-  return /** @type { MultiHasher } */({
-    multi: true,
-    update(data) {
-      for (const hash of hashers) {
-        hash.update(data);
-      }
-      return this;
-    },
-    digest(encoding) {
-      const result = {};
-      for (let i = 0; i < hashers.length; i++) {
-        hasherPending.set(hashers[i], false);
-        result[algoNames[i]] = hashers[i].digest(encoding);
-      }
-      return result;
-    }
-  });
-}
-
-/**
- * Create a hash instance that implements WritableStream.
- * @overload
- * @param {string} algorithm One or more digest algorithm names
- * @return {Promise<WritableStreamHasher>}
- */
-/**
- * @overload
- * @param {string[]} algorithm One or more digest algorithm names
- * @return {Promise<WritableStreamMultiHasher>}
- */
-async function createStream(algorithm = 'sha512') {
-  const hash = await (Array.isArray(algorithm) ? createMultiHasher(algorithm) : createHasher(algorithm));
-  /** @type {any} */
-  const ws = new WritableStream({
-    write(chunk) {
-      hash.update(chunk);
-    }
-  });
-  ws.update = function (data) {
-    hash.update(data);
-  };
-  ws.digest = function (encoding) {
-    if (!isMulti(hash)) hasherPending.set(hash, false);
-    return hash.digest(encoding);
-  }
-  return ws;
-}
-
-/** @typedef {TransformStream & { digest: IMultiDigest }} StreamThroughHash */
-/**
- * Create a pass through stream that pass data as it is and calculate the digest as data pass through.
- * @param {string|string[]} [algorithm] 
- * @param {object} [options]
- * @return {Promise<StreamThroughHash>}
- */
-async function createStreamThrough(algorithm = 'sha512', options) {
-  const hash = await (Array.isArray(algorithm) ? createMultiHasher(algorithm) : createHasher(algorithm));
-  const { writableStrategy, readableStrategy } = options || {};
-  //hash.setEncoding('hex');
-  /** @type {any} */
-  const ts = new TransformStream({
-    transform(chunk, controller) {
-      controller.enqueue(chunk);
-      hash.update(chunk);
-    }
-  }, writableStrategy, readableStrategy);
-  ts.digest = function (encoding) {
-    if (!isMulti(hash)) hasherPending.set(hash, false);
-    return hash.digest(encoding);
-  }
-  return ts;
-}
-
-/**
- * 
- * @overload
- * @param {string[]} algorithm 
- * @param {IDataType | ReadableStream} input 
- * @return {Promise<{[key:string]: string}>}
- */
-/**
- * @overload
- * @param {string} algorithm 
- * @param {IDataType | ReadableStream} input 
- * @return {Promise<string>}
- */
-async function digest(algorithm, input) {
-  if (input instanceof ReadableStream) {
-    const hash = await createStream(algorithm);
-    await input.pipeTo(hash);
-    return hash.digest();
-  } else {
-    const hash = await (Array.isArray(algorithm) ? createMultiHasher(algorithm) : createHasher(algorithm));
-    if (!isMulti(hash)) hasherPending.set(hash, false);
-    return hash.update(input).digest();
-  }
-}
-
-async function initSync(algorithm) {
-  if (!hasherSyncCache[algorithm]) {
-    const factory = hasherFactory[algorithm];
-    if (!factory) throw new Error(`Unsupported digest algorithm: ${algorithm}`);
-    const hasher = await factory();
-    hasherSyncCache[algorithm] = hasher;
-  }
-}
-
-/**
- * For this sync function to work correctly, it requires a call to initSync() to be awaited for each algorithm before calling this function. 
- * @param {string} algorithm 
- * @param {IDataType} input 
- */
-function digestSync(algorithm, input) {
-  const hasher = hasherSyncCache[algorithm];
-  if (!hasher) throw new Error(`Hasher for algorithm ${algorithm} is not initialized. Please call and await initSync('${algorithm}') before calling this function.`);
-  return hasher.init().update(input).digest();
-}
-
-const HEX_DIGEST_LENGTH = {
-  'md5': 32,
-  'sha1': 40,
-  'sha256': 64,
-  'sha512/256': 64,
-  'sha512': 128,
-  'blake2b-160': 40,
-  'blake2b-256': 64,
-  'blake2b-384': 96,
-  'blake2b-512': 128,
-  'crc32': 8,
-  'size': 1
-};
-
-function getHexDigestLength(algorithm) {
-  return HEX_DIGEST_LENGTH[algorithm] || 0;
-}
-
-const OcflDigest = {
-  CONTENT,
-  FIXITY,
-  // algorithms,
-  createStream, createStreamThrough, digest,
-  registerHasherFactory, hasAlgorithm, getHexDigestLength,
-  digestSync, initSync
-};
-
-module.exports = {
-  OcflDigest,
-  [testSymbol]: {
-    hasherCache
-  }
-  // DIGEST,
-  // DIGEST_FIXITY,
-  // digestSync,
-  // digestAsync,
-  // digestFromStream,
-  // digestStream,
-  // HashThrough
-};