@ocfl/ocfl 0.1.1 → 0.2.0

package/README.md

@@ -1,7 +1,8 @@
 # ocfl-js - Common modules
 This is JavaScript/Node.js library to create and interact with Oxford Common File Layout (OCFL) storage and objects within it. 
 This package contains abstract classes definition. Do not use this package directly. To work with an actual OCFL storage and object, use the following concrete implementation:
-* ocfl-fs - File system storage backend
+
+* ocfl-fs - File system backend
 
 ## Extensions
 This library by default includes all the community extensions defined in the [OCFL Community Extensions page](https://github.com/OCFL/extensions). The implementation is organised in the ocfl-extensions package. 
@@ -14,3 +15,7 @@ To implement a local extension in this library, first follow the description in
        
     }
 
+
+## Implementation
+
+A new data store backend can be implemented by creating a new package with a store class that extends the `OcflStore` class and implements its methods. Please see the source code of ocfl-fs as an example implementation

package/index.js

@@ -21,11 +21,12 @@ const { OcflConstants, OcflDigest, OcflExtension, OcflStorageLayout, OcflStore,
 
 /**
  * @typedef {Object} StorageLayout
- * @property {typeof extensions.FlatDirectStorageLayout} [FlatDirectStorageLayout]
- * @property {typeof extensions.FlatOmitPrefixStorageLayout} [FlatOmitPrefixStorageLayout]
- * @property {typeof extensions.HashAndIdNTupleStorageLayout} [HashAndIdNTupleStorageLayout]
- * @property {typeof extensions.HashedNTupleStorageLayout} [HashedNTupleStorageLayout]
- * @property {typeof extensions.NTupleOmitPrefixStorageLayout} [NTupleOmitPrefixStorageLayout]
+ * @property {typeof extensions.FlatDirectStorageLayout} FlatDirectStorageLayout
+ * @property {typeof extensions.FlatOmitPrefixStorageLayout} FlatOmitPrefixStorageLayout
+ * @property {typeof extensions.HashAndIdNTupleStorageLayout} HashAndIdNTupleStorageLayout
+ * @property {typeof extensions.HashedNTupleStorageLayout} HashedNTupleStorageLayout
+ * @property {typeof extensions.NTupleOmitPrefixStorageLayout} NTupleOmitPrefixStorageLayout
+ * @property {typeof extensions.PathDirectStorageLayout} PathDirectStorageLayout
  */
 /** @typedef {import('./lib/extension.js').OcflExtensionConfig} OcflExtensionConfig */
 
@@ -73,7 +74,7 @@ class Ocfl {
   get StorageLayout() { return OcflStorageLayout.layout }
 
   /**
-   * Create a storage layout from extension name or config
+   * Instantiate a storage layout from extension name or config
    * @param {string|OcflExtensionConfig} config - Layout name or config
    */
   storageLayout(config) {
@@ -90,7 +91,7 @@ class Ocfl {
   }
 
   /**
-   * Create an OCFL Object
+   * Instantiate an OCFL Object object
    * @param {OcflObjectConfig} config
    * @param {SO} [storeOptions]
    * @return {OcflObject}
@@ -100,7 +101,7 @@ class Ocfl {
   }
 
   /**
-   * Create an OCFL Storage
+   * Instantiate an OCFL Storage object
    * @param {OcflStorageConfig} config
    * @param {SO} [storeOptions]
    * @return {OcflStorage}
@@ -109,16 +110,27 @@ class Ocfl {
     return new OcflStorageImpl(config, this.#store.getInstance(storeOptions));
   }
 
-}
-
-class Storage {
-  async create(config) {
-
+  /**
+   * Create a new OCFL Storage
+   * @param {OcflStorageConfig} config
+   * @param {SO} [storeOptions]
+   * @return {Promise<OcflStorage>}
+   */
+  async createStorage(config, storeOptions = this.#defaultOptions) {
+    return (new OcflStorageImpl(config, this.#store.getInstance(storeOptions))).create();
   }
-  async load(config) {
 
+  /**
+   * Load an existing OCFL Storage
+   * @param {OcflStorageConfig} config
+   * @param {SO} [storeOptions]
+   * @return {Promise<OcflStorage>}
+   */
+  async loadStorage(config, storeOptions = this.#defaultOptions) {
+    return (new OcflStorageImpl(config, this.#store.getInstance(storeOptions))).load();
   }
-} 
+}
+
 
 
 module.exports = {

package/lib/extension.js

@@ -7,7 +7,7 @@ const { NotImplementedError } = require("./error");
 const extensionByName = {};
 /** Index by extension class name */
 const extensionByClassName = {};
-/** @type {Object.<string, typeof OcflStorageLayout>} */
+
 const storageLayout = {};
 
 /** @typedef {{ extensionName?: string;[key: string]: any; }} OcflExtensionConfig */
@@ -110,7 +110,7 @@ class OcflExtension {
    */
   constructor(config, defaultConfig) {
     /** @type {C} */
-    this.parameters = defaultConfig ? Object.create(/**@type{object}*/(defaultConfig)) : {};
+    this.parameters = typeof defaultConfig === 'object' ? Object.create(/**@type{object}*/(defaultConfig)) : {};
     if (typeof config === 'object') {
       for (let k in config) {
         if (k in defaultConfig) this.parameters[k] = config[k];
@@ -157,7 +157,7 @@ class OcflExtension {
 class OcflStorageLayout extends OcflExtension {
 
   static get layout() {
-    return storageLayout;
+    return /** @type StorageLayout */(storageLayout);
   }
 
   /**

package/lib/extensions/000N-path-direct-storage-layout.js

@@ -0,0 +1,61 @@
+const { OcflStorageLayout } = require("../extension");
+const { OcflDigest } = require("../digest");
+const path = require("path");
+
+const DefaultConfig = {
+  extensionName: '000N-path-direct-storage-layout',
+  omitScheme: false,
+  /** @type {any} */
+  replace: [],
+  suffix: '/__object__'
+};
+
+/** @typedef {Partial<typeof DefaultConfig>} PathDirectStorageLayoutConfig */
+/**
+ * @extends {OcflStorageLayout<typeof DefaultConfig>}
+ */
+class PathDirectStorageLayout extends OcflStorageLayout {
+  static get NAME() { return DefaultConfig.extensionName; }
+  static get DESCRIPTION() {
+    return "OCFL object identifiers are used directly as the path.";
+  }
+
+  /**
+   * 
+   * @param {PathDirectStorageLayoutConfig} [config] 
+   */
+  constructor(config) {
+    super(config, DefaultConfig);
+  }
+
+  /**
+   * @param {string} id
+   * @return {string} 
+   */
+  map(id) {
+    var suffix = this.parameters.suffix;
+    if (suffix && id.endsWith(suffix)) throw new Error(`Identifier cannot end with '${suffix}'`);
+    var p = id ?? '';
+    for (const r of this.parameters.replace) {
+      p = p.replace(r[0], r[1]);
+    }
+    try {
+      let u = new URL(p);
+      let parts = [];
+      if (!this.parameters.omitScheme && u.protocol !== 'file:') {
+        parts.push(u.protocol.replace(':', ''));
+      }
+      if (u.hostname) {
+        parts.push(u.hostname.replace(',', '_').replace(';', '/'));
+      }
+      p = path.join(parts.join('_'), u.pathname + u.search + u.hash);
+    } catch (error) {
+    }
+    p = p.replace(/^\/+/, '').replace(/\/+$/, '');
+    return path.join(p, this.parameters.suffix);
+    //if (/^\w+:/.test(id)) {
+  }
+
+}
+
+module.exports = { PathDirectStorageLayout };

package/lib/extensions/index.js

@@ -6,7 +6,8 @@ module.exports = {
   ...require('./0004-hashed-n-tuple-storage-layout'),
 //  ...require('./0005-mutable-head'),
   ...require('./0006-flat-omit-prefix-storage-layout'),
-  ...require('./0007-n-tuple-omit-prefix-storage-layout')
+  ...require('./0007-n-tuple-omit-prefix-storage-layout'),
+  ...require('./000N-path-direct-storage-layout')
   //...require('./0008-schema-registry')
 };
 //const { DigestAlgorithm } = require('./0001-digest-algorithms.js');

package/lib/object.js

@@ -123,13 +123,19 @@ class OcflObject {
    * Identifier of the OCFL Object
    * @return {string} 
    */
-  get id() { throw new NotImplementedError() }
+  get id() { throw new NotImplementedError(); }
 
   /** 
-  * The non-storage specific absolute path to the root of this OCFL Object
+  * The absolute path to the root of this OCFL Object
   * @return {string} 
   */
-  get root() { throw new NotImplementedError() }
+  get root() { throw new NotImplementedError(); }
+
+  /** 
+  * The absolute path to the temporary workspace of this OCFL Object
+  * @return {string} 
+  */
+  get workspace() { throw new NotImplementedError(); }
 
   toString() { return `OcflObject { root: ${this.root}, id: ${this.id} }`; }
 
@@ -140,7 +146,7 @@ class OcflObject {
    * @param {string} [version] - Full name of the version, eg: v1, v2, v3  
    * @returns {Promise<OcflObjectInventory>}
    */
-  async getInventory(version = 'latest') { throw new NotImplementedError() }
+  async getInventory(version = 'latest') { throw new NotImplementedError(); }
 
   /**
    * Update the content files or directories as one transaction and commit the changes as a new version.
@@ -150,7 +156,7 @@ class OcflObject {
    * @param {ObjectUpdateMode} [mode=UPDATE_MODE.MERGE]
    * @return {Promise<?OcflObjectTransaction>}
    */
-  async update(updater, mode = UPDATE_MODE.MERGE) { throw new NotImplementedError() }
+  async update(updater, mode) { throw new NotImplementedError(); }
 
   /**
    * Import one or more content files or directories to the object 
@@ -199,7 +205,8 @@ class OcflObject {
 
   /**
    * Iterate through the content files contained in the specified version. 
-   * Returns an array of FileRef which consists of logical path, content path, and digest, in no particular order
+   * Returns an Iterator that contains FileRef which consists of logical path, content path, and digest, in no particular order.
+   * The iterator is implemented as a generator.
    * @param {string} [version]
    * @return {Promise<Generator<OcflObjectFile, void, unknown>>}
    */
@@ -224,7 +231,7 @@ class OcflObject {
    * @type { GetFileByLogical & GetFileByOpt }
    */
   // @ts-ignore
-  getFile = function (opt, version) { }
+  getFile = function (opt, version) { };
 
   /**
    * Count the number of files contained in this object
@@ -258,6 +265,8 @@ class OcflObjectImpl extends OcflObject {
   /** @type {string} */
   #root;
   /** @type {string} */
+  #workspace;
+  /** @type {string} */
   #id;
   /** @type {OcflExtension[]} */
   #extensions;
@@ -283,13 +292,14 @@ class OcflObjectImpl extends OcflObject {
     this.digestAlgorithm = DIGEST.of(config.digestAlgorithm || 'sha512');
     if (!this.digestAlgorithm) throw new Error('Invalid digest algorithm. Must be one of `sha256` or `sha512`.');
     //this.fixityDigest = config.fixityDigest ?? this.digestAlgorithm;
-    this.workspace = config.workspace ? path.resolve(config.workspace) : undefined;
+    this.#workspace = config.workspace ? path.resolve(config.workspace) : undefined;
     this.contentDirectory = config.contentDirectory ?? 'content';
     /** @type Object.<string, OcflObjectInventory> */
     this._inventory = {};
   }
   get id() { return this._inventory?.latest?.id || this.#id; }
   get root() { return this.#root; }
+  get workspace() { return this.#workspace; }
 
   async getInventory(version = 'latest') {
     if (!this._inventory[version]) {
@@ -324,7 +334,7 @@ class OcflObjectImpl extends OcflObject {
       OcflDigest.digestAsync(data.digestAlgorithm, datastr)
     ]));
     digest = digest.match(/[^\s]+/)?.[0];
-    if (digest !== actualDigest) throw new Error('corrupted file');
+    if (digest !== actualDigest) throw new Error(`Inventory file ${this.root}/${invPath} digest mismatch: recorded=${digest} actual=${actualDigest}`);
     //return new OcflObjectInventory({ data, digest });
     return new OcflObjectInventory(data);
   }
@@ -373,13 +383,13 @@ class OcflObjectImpl extends OcflObject {
    */
   readFile = async function (relPath, options) {
     return this.#store.readFile(path.join(this.root, relPath), options);
-  }
+  };
 
   getFile = function (opt, version) {
     var obj = this;
     var opt_ = typeof opt === 'string' ? { logicalPath: opt, version: version } : opt;
     return new OcflObjectFile(this, opt_);
-  }
+  };
 
   // async _resolveContentPath({ logicalPath = '', contentPath = '', digest = '', version = '' }) {
   //   if (!contentPath) {
@@ -477,6 +487,7 @@ class OcflObjectImpl extends OcflObject {
       // workspace dir exists, abort update
       throw new Error('Uncommitted changes detected. Object is being updated by other process or there has been a failed update attempt');
     }
+    await this._validateObjectPath();
     let createdDir = await this.#store.mkdir(workspacePath);
     if (!this.workspace) await this._ensureNamaste();
     return createTransactionProxy(new OcflObjectTransactionImpl(this, this.#store, inventory, workspaceVersionPath, createdDir));
@@ -491,18 +502,28 @@ class OcflObjectImpl extends OcflObject {
       return;
     }
     // namaste not found, check if object root is not empty
+    let files;
     try {
-      let files = await this.#store.readdir(this.root);
-      if (files.length > 0) throw new Error('Cannot create an OCFL Object in a non-empty directory');
+      files = await this.#store.readdir(this.root);
     } catch (error) {
     }
+    if (files && files.length > 0) throw new Error('Cannot create an OCFL Object in a non-empty directory');
     //create the namaste file
     let filePath = path.join(this.root, NAMASTE_T + prefix + this.ocflVersion);
     await this.#store.writeFile(filePath, prefix + this.ocflVersion + '\n', 'utf8');
   }
 
+  // all the parent directories cannot be an ocfl object
+  async _validateObjectPath() {
+    var p = this.root;
+    while (p !== '.' && p !== '/' ) {
+      p = path.dirname(p);
+      let nv = await findNamasteVersion(this.#store, NAMASTE_PREFIX_OBJECT, p);
+      if (nv) throw new Error(`Object root cannot be nested under another object ${p}`);
+    }
+  }
+  
   async isObject(aPath) { }
-  async importDir(id, sourceDir) { }
   getVersionString(i) { }
   async determineVersion() { }
   nameVersion(version) {
@@ -540,7 +561,7 @@ const proxyHandler = {
       }
     }
   }
-}
+};
 
 /**
  * Return any OcflObject implementation as OcflObject.

package/lib/storage.js

@@ -7,8 +7,8 @@ const { OcflStorageLayout } = require("./extension");
 const { HashedNTupleStorageLayout } = require("./extensions/0004-hashed-n-tuple-storage-layout");
 const { OcflObject, OcflObjectImpl } = require("./object");
 const { isDirEmpty, findNamasteVersion } = require("./utils.js");
-const { OCFL_VERSION, OCFL_VERSIONS, OCFL_LAYOUT, 
-  EXTENSIONS_DIR, EXTENSION_CONFIG, 
+const { OCFL_VERSION, OCFL_VERSIONS, OCFL_LAYOUT,
+  EXTENSIONS_DIR, EXTENSION_CONFIG,
   NAMASTE_PREFIX_STORAGE, NAMASTE_PREFIX_OBJECT, NAMASTE_T } = require('./constants').OcflConstants;
 const DIGEST = require('./digest').OcflDigest.CONTENT;
 
@@ -66,42 +66,44 @@ class OcflStorage {
    * Create the storage root in the underlying storage backend
    * @param {Object} [options] - Additional options for creating storage root
    * @param {boolean} [options.ocflSpecText=false] - If true, create a copy of the OCFL specification text in the storage root
+   * @return {Promise<OcflStorage>}
    */
   async create(options) { throw new NotImplementedError(); }
 
-  /**
-   * Check if the storage root path points to an existing file or non-empty directory in the underlying backend store.
-   * The existing directory may or may not be a valid OCFL Storage.
-   * @return {Promise<boolean>}
-   */
-  async exists() { throw new NotImplementedError(); }
-
   /**
    * Check namaste, retrieve storage layout extension, and read its config file.
-   * @return {Promise<boolean>}
+   * @return {Promise<OcflStorage>}
    */
   async load() { throw new NotImplementedError(); }
 
+  /**
+  * Check if the storage root path points to an existing file or non-empty directory in the underlying backend store.
+  * The existing directory may or may not be a valid OCFL Storage.
+  * @return {Promise<boolean>}
+  */
+  async exists() { throw new NotImplementedError(); }
+
   /**
    * 
-   * @return {AsyncIterator<OcflObject>}
+   * @return {AsyncIterableIterator<OcflObject>}
    */
   [Symbol.asyncIterator]() { throw new NotImplementedError(); }
 
   /**
-   * Return all OCFL objects under this storage as array
-   * @return {Promise<OcflObject[]>}
+   * Return a new AsyncIterator that contains all the OCFL objects in this Storage.
+   * @return {AsyncIterableIterator<OcflObject>}
    */
-  async objects() {
-    const objects = []; 
-    for await(const o of this) objects.push(o); 
-    return objects;
+  objects() {
+    return this[Symbol.asyncIterator]();
+    // const objects = [];
+    // for await (const o of this) objects.push(o);
+    // return objects;
   }
 
   async delete(id) {
     return this.remove(id);
   }
-  
+
   async purge(id) {
     return this.remove(id);
   }
@@ -188,7 +190,7 @@ class OcflStorageImpl extends OcflStorage {
   object(opt) {
     let id, root;
     if (typeof opt === 'string') id = opt;
-    else ({id, root} = opt);
+    else ({ id, root } = opt);
     let relObjectRoot = root || this.objectRoot(id);
     return new OcflObjectImpl({
       id,
@@ -230,7 +232,7 @@ class OcflStorageImpl extends OcflStorage {
 
     //@todo: load storage level extensions
 
-    return true;
+    return this;
   }
 
   /**
@@ -264,11 +266,9 @@ class OcflStorageImpl extends OcflStorage {
     if (options?.ocflSpecText) {
 
     }
+    return this;
   }
 
-  /**
-   * @return {AsyncIterator<OcflObject, OcflObject>}
-   */
   [Symbol.asyncIterator]() {
     let store = this.#store;
     let workspace = this.#workspace;
@@ -276,6 +276,7 @@ class OcflStorageImpl extends OcflStorage {
     let root = this.root;
     let stack = [this.#store.opendir(this.root)];
     return {
+      [Symbol.asyncIterator]() { return this; },
       async next() {
         let dirp;
         while ((dirp = stack.pop())) {

package/lib/types.d.ts

@@ -42,3 +42,12 @@ interface ReadFile {
       | BufferEncoding
   ): Promise<string>;
 }
+
+type StorageLayout = {
+  FlatDirectStorageLayout: typeof import('./extensions/0002-flat-direct-storage-layout').FlatDirectStorageLayout
+  HashAndIdNTupleStorageLayout: typeof import('./extensions/0003-hash-and-id-n-tuple-storage-layout').HashAndIdNTupleStorageLayout
+  HashedNTupleStorageLayout: typeof import('./extensions/0004-hashed-n-tuple-storage-layout').HashedNTupleStorageLayout
+  FlatOmitPrefixStorageLayout: typeof import('./extensions/0006-flat-omit-prefix-storage-layout').FlatOmitPrefixStorageLayout
+  NTupleOmitPrefixStorageLayout: typeof import('./extensions/0007-n-tuple-omit-prefix-storage-layout').NTupleOmitPrefixStorageLayout
+  PathDirectStorageLayout: typeof import('./extensions/000N-path-direct-storage-layout').PathDirectStorageLayout
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ocfl/ocfl",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Oxford Common File Layout (OCFL) JS library",
   "main": "index.js",
   "exports": {
@@ -33,11 +33,13 @@
     "Peter Sefton (https://orcid.org/0000-0002-3545-944X)"
   ],
   "license": "GPL-3.0-or-later",
+  "dependencies": {
+    "blake2": "^4.1.1"
+  },
   "devDependencies": {
     "@types/mocha": "^9.1.1",
     "@types/node": "^17.0.33",
-    "clean-jsdoc-theme": "^3.3.4",
-    "jsdoc": "^3.6.10",
+    "typedoc": "^0.23.15",
     "mocha": "^10.0.0"
   },
   "mocha": {
@@ -48,9 +50,6 @@
     ],
     "recursive": true
   },
-  "dependencies": {
-    "blake2": "^4.1.1"
-  },
   "bugs": {
     "url": "https://github.com/Arkisto-Platform/ocfl-js/issues"
   },
@@ -59,5 +58,10 @@
     "lib": "lib",
     "test": "test"
   },
-  "files": [ "index.js", "index.mjs", "lib", "resources" ]
+  "files": [
+    "index.js",
+    "index.mjs",
+    "lib",
+    "resources"
+  ]
 }