@ocfl/ocfl 0.2.0 → 0.3.0

package/lib/storage.js

@@ -5,19 +5,23 @@ const validation = require('./validation.js');
 const { NotImplementedError } = require("./error");
 const { OcflStorageLayout } = require("./extension");
 const { HashedNTupleStorageLayout } = require("./extensions/0004-hashed-n-tuple-storage-layout");
-const { OcflObject, OcflObjectImpl } = require("./object");
+const { OcflObject } = require("./object");
 const { isDirEmpty, findNamasteVersion } = require("./utils.js");
 const { OCFL_VERSION, OCFL_VERSIONS, OCFL_LAYOUT,
   EXTENSIONS_DIR, EXTENSION_CONFIG,
   NAMASTE_PREFIX_STORAGE, NAMASTE_PREFIX_OBJECT, NAMASTE_T } = require('./constants').OcflConstants;
-const DIGEST = require('./digest').OcflDigest.CONTENT;
-
+const { OcflDigest } = require('./digest');
 
 const DEFAULT_LAYOUT = HashedNTupleStorageLayout;
 
-/** 
- * @typedef {import('./extension').OcflExtensionConfig} OcflExtensionConfig
- * @typedef {import('./store').OcflStore} OcflStore
+/** @typedef {import('./store').OcflStore} OcflStore */
+/** @typedef {import('./types.js').OcflExtensionConfig} OcflExtensionConfig */
+/** @typedef {import('./types.js').OcflVersion} OcflVersion */
+
+
+/**
+ * An abstraction to represent the OCFL Storage Root, which is
+ * a base directory used to store OCFL Objects, identified by a [NAMASTE] file “0=ocfl_1.1”.
  */
 
 class OcflStorage {
@@ -120,13 +124,13 @@ class OcflStorage {
  * @property {('sha256' | 'sha512')} [digestAlgorithm] - Digest algorithm for content-addressing, must use either sha512 or sha256
  * @property {string} [contentDirectory='content'] - Content directory name. Only applies to a newly created object.
  * @property {string} [ocflVersion=c.OCFL_VERSION] - Ocfl version. Only applies to a newly created object.
+ * @property {string[]} [fixityAlgorithms] - A default list of digest algorithm names to be added to the fixity block of objects under this storage.
  */
 
 /**
  * General implementation of {@link OcflStorage} that can uses different datastore backends. 
  * This class provides common functionalities for the subclasses and 
  * at the same provide encapsulation emulating private and protected methods.
- * @implements {OcflStorage}
  */
 class OcflStorageImpl extends OcflStorage {
 
@@ -156,7 +160,9 @@ class OcflStorageImpl extends OcflStorage {
     if (this.#workspace && this.#workspace.startsWith(this.#root + path.sep) && !this.#workspace.startsWith(this.#root + path.sep + 'extensions' + path.sep)) {
       throw new Error('[OcflStorage] config.workspace cannot be the same as or a subpath of config.root');
     }
-    this.#layout = layout instanceof OcflStorageLayout ? layout : this.#createLayout(layout);
+    if (layout) {
+      this.#layout = layout instanceof OcflStorageLayout ? layout : this.#createLayout(layout);
+    }
     this.ocflVersion = ocflVersion || OCFL_VERSION;
     this.#objectConfig = objectConfig;
   }
@@ -192,15 +198,14 @@ class OcflStorageImpl extends OcflStorage {
     if (typeof opt === 'string') id = opt;
     else ({ id, root } = opt);
     let relObjectRoot = root || this.objectRoot(id);
-    return new OcflObjectImpl({
+    return new OcflObject({
       id,
       root: path.join(this.root, relObjectRoot),
       // @todo:use extension workspace if not defined
       workspace: this.#workspace ? path.join(this.#workspace, relObjectRoot) : undefined,
-      ocflVersion: this.ocflVersion,
+      ocflVersion: /** @type {OcflVersion} */(this.ocflVersion),
       ...this.#objectConfig
-    },
-      this.#store);
+    }, this.#store);
   }
 
   async exists() {
@@ -229,7 +234,8 @@ class OcflStorageImpl extends OcflStorage {
     } catch (error) {
     }
     this.#layout = this.#createLayout(layoutConfig || layoutName);
-
+    const layoutDigestAlgorithm = this.#layout.get('digestAlgorithm');
+    if (layoutDigestAlgorithm) await OcflDigest.initSync(layoutDigestAlgorithm);
     //@todo: load storage level extensions
 
     return this;
@@ -256,6 +262,8 @@ class OcflStorageImpl extends OcflStorage {
     if (!this.#layout) {
       this.#layout = new DEFAULT_LAYOUT();
     }
+    const layoutDigestAlgorithm = this.#layout.get('digestAlgorithm');
+    if (layoutDigestAlgorithm) await OcflDigest.initSync(layoutDigestAlgorithm);
     let layout = { extension: this.#layout.name, description: this.#layout.description };
     await this.#store.writeFile(path.join(this.root, OCFL_LAYOUT), JSON.stringify(layout, null, 2), 'utf8');
     if (this.#layout.config) {
@@ -269,6 +277,9 @@ class OcflStorageImpl extends OcflStorage {
     return this;
   }
 
+  /**
+   * Allow async iteration over all OCFL objects in this storage.
+   */
   [Symbol.asyncIterator]() {
     let store = this.#store;
     let workspace = this.#workspace;
@@ -297,10 +308,10 @@ class OcflStorageImpl extends OcflStorage {
           if (nv) {
             //ocfl object exists
             // inventory exists
-            let object = new OcflObjectImpl({
+            let object = new OcflObject({
               root: basePath,
               workspace: workspace ? path.join(workspace, path.relative(root, basePath)) : undefined,
-              ocflVersion: nv,
+              ocflVersion: /** @type {OcflVersion} */(nv),
               ...config
             }, store);
             //await object.load();

package/lib/store.js

@@ -1,6 +1,5 @@
 const path = require('path');
-const { pipeline } = require('stream/promises');
-const { parallelize, dataSourceAsIterable } = require('./utils');
+const { parallelize, joinTypedArray } = require('./utils');
 
 const emptyOptions = {};
 
@@ -9,6 +8,7 @@ const emptyOptions = {};
  * such as local file system and cloud storage
  */
 class OcflStore {
+  /** @type {Map<Object, OcflStore>} */
   static instances;
 
   /**
@@ -52,6 +52,7 @@ class OcflStore {
   * @param {string} filePath - Absolute path
   * @param {*} options 
   * @return {Promise<import('fs').ReadStream>}
+  * @deprecated Use {@link createReadable()} instead
   */
   async createReadStream(filePath, options) { throw new Error('Not Implemented'); }
 
@@ -60,30 +61,50 @@ class OcflStore {
    * @param {string} relPath 
    * @param {*} options 
    * @return {Promise<import('fs').WriteStream>}
+   * @deprecated Use {@link createWritable()} instead
    */
   async createWriteStream(relPath, options) { throw new Error('Not Implemented'); }
 
+  /**
+  * Create a ReadableStream to get the content of a file
+  * @param {string} filePath - Absolute path
+  * @param {*} options 
+  * @return {Promise<ReadableStream>}
+  */
+  async createReadable(filePath, options) { throw new Error('Not Implemented'); }
+
+  /**
+   * Create a WritableStream that writes data to {@link relPath}. 
+   * @param {string} relPath 
+   * @param {*} options 
+   * @return {Promise<WritableStream>}
+   */
+  async createWritable(relPath, options) { throw new Error('Not Implemented'); }
+
   /**
    * Provide a common interface to read a file inside an object. 
    * A concrete subclass MAY implement this method to provide read access to its underlying storage backend.
-   * The default implementation uses the {@linkcode OcflStore#createReadStream} method.
+   * The default implementation uses the {@linkcode OcflStore#createReadable} method.
    * @param {string} filePath 
    * @param {*} [options] 
    */
   async readFile(filePath, options) {
-    let rs = await this.createReadStream(filePath, options);
-    let chunks = [];
+    const rs = await this.createReadable(filePath, options);
+    let buffer = (typeof options === 'string' || options.encoding) ? '' : new Uint8Array(new ArrayBuffer(0, { maxByteLength: 16 * 1024 }));
     for await (const chunk of rs) {
-      chunks.push(chunk);
-    }
-    if (typeof options === 'string' || options.encoding) {
-      return Buffer.concat(chunks);
-    } else {
-      return chunks.join('');
+      if (typeof chunk === 'string') {
+        if (typeof buffer !== 'string') buffer = new TextDecoder().decode(buffer);
+        buffer += chunk;
+      } else {
+        if (typeof buffer === 'string') buffer = new TextEncoder().encode(buffer);
+        // @ts-ignore
+        buffer = joinTypedArray(buffer, chunk);
+      }
     }
+    return buffer;
   }
 
-  /** @typedef {string|Buffer|NodeJS.ArrayBufferView|AsyncIterable|Iterable|import('stream').Readable} WriteFileData */
+  /** @typedef {string|Buffer|Uint8Array|DataView|AsyncIterable|Iterable|ReadableStream} WriteFileData */
   /**
    * Provide a common interface to write a file inside an object. 
    * A concrete subclass MAY implement this method to provide write access to its underlying storage backend.
@@ -93,10 +114,15 @@ class OcflStore {
    * @param {Object|string} options 
    */
   async writeFile(filePath, data, options) {
-    var source = dataSourceAsIterable(data);
     await this.mkdir(path.dirname(filePath));
-    const target = await this.createWriteStream(filePath, options);
-    await pipeline(source, target);
+    const target = await this.createWritable(filePath, options);
+    if (data instanceof ReadableStream) {
+      await data.pipeTo(target);
+    } else {
+      const w = target.getWriter();
+      await w.write(data);
+      await w.close();
+    }
   }
 
   /**
@@ -106,7 +132,7 @@ class OcflStore {
    * @param {string} target - destination filename of the copy operation
    */
   async copyFile(source, target) {
-    await this.writeFile(target, await this.createReadStream(source));
+    await this.writeFile(target, await this.createReadable(source));
   }
 
   /**
@@ -125,6 +151,18 @@ class OcflStore {
    */
   async readdir(filePath, options) { throw new Error('Not Implemented'); }
 
+  /**
+   * List all files contained in a directory including the metadata such as size and last modified date.
+   * The returned file.path is a relative path to the dirPath
+   * @param {string} dirPath 
+   * @param {object} [options]
+   * @param {BufferEncoding} [options.encoding]
+   * @param {number} [options.bufferSize]
+   * @param {boolean} [options.recursive]
+   * @return {Promise<AsyncIterableIterator<{name: string, path: string, size: number, lastModified: Date}>> }
+   */
+  async list(dirPath, { encoding='utf8', bufferSize=32, recursive=false} = {}) { throw new Error('Not Implemented'); }
+
   /**
   * Recursively create a directory if the storage backend supports it, otherwise do nothing
   * @param {string} filePath - The directory path