@ocfl/ocfl 0.1.0

package/lib/temp.js

@@ -0,0 +1,264 @@
+const fs = require('fs-extra');
+const path = require('path');
+const hasha = require('hasha');
+const uuidv5 = require('uuid/v5');
+const _ = require('lodash');
+const DIGEST_ALGORITHM = 'sha512';
+
+class OcflObject {
+
+
+  constructor(path) {
+    this.path = path;
+    this.ocflVersion = '1.0';
+    this.contentVersion = null; // No content yet
+    this.id = null; // Not set yet
+    this.DIGEST_ALGORITHM = DIGEST_ALGORITHM;
+
+  }
+
+  async writeInventories(inv, version) {
+    const main_inv = await fs.writeJson(path.join(this.path, 'inventory.json'), inv, { spaces: 2 });
+    const version_inv = await fs.writeJson(path.join(this.path, version, 'inventory.json'), inv, { spaces: 2 });
+    const inv_hash = await this.hash_file(path.join(this.path, 'inventory.json'))
+    const digest_file_v1 = await fs.writeFile(path.join(this.path, version, 'inventory.json.' + DIGEST_ALGORITHM), inv_hash + "   inventory.json");
+    const digest_file = await fs.writeFile(path.join(this.path, 'inventory.json.' + DIGEST_ALGORITHM), inv_hash + "   inventory.json");
+
+  }
+
+  // addContent is passed an id and a callback - this takes one argument, the
+  // directory into which content is to be written.
+  // This is called for both new and merged versions: the incoming version is
+  // written as v1 of a new object in the deposit directory, and if it's not the
+  // first version it's then merged with the existing most recent version
+
+  async addContent(id, writeContent) {
+    // Copy files into v1
+
+    const version = "v1" // Always a fresh start as we're not touching an existing repo object
+    const versionPath = path.join(this.path, version, "content");
+    await fs.ensureDir(versionPath);
+    await writeContent(versionPath);
+
+    // Make an inventory
+    const inv = await this.inventory(id, versionPath);
+
+    // Put the inventory in the root AND version dir
+    await this.writeInventories(inv, version)
+    this.contentVersion = await this.determineVersion();
+    this.id = id;
+  }
+
+  // preserving the old interface here
+
+  async importDir(id, sourceDir) {
+    await this.addContent(id, async (targetDir) => {
+      await fs.copy(sourceDir, targetDir);
+    })
+  }
+
+
+  async create(path) {
+    // Creates a blank object with a content dir but no content at <path>
+    if (this.path) {
+      throw new Error("This object has already been initialized at: " + this.path)
+    }
+    this.path = path;
+    const stats = await fs.stat(this.path);
+    if (await fs.pathExists(this.path) && stats.isDirectory()) {
+      const readDir = await fs.readdir(this.path);
+      if (readDir.length <= 0) { // empty so initialise an object here
+        const generateNamaste = await this.generateNamaste(this.path, this.ocflVersion);
+      } else {
+          throw new Error('can\'t initialise an object here as there are already files')
+        }
+    } else {
+      //else if it doesnt it dies
+      throw new Error('directory does not exist');
+    }
+  }
+
+  async load(path) {
+    // Tries to load an existing object residing at <path>
+    if (this.path) {
+      throw new Error("This object has already been initialized at: " + this.path)
+    }
+    this.path = path;
+    const stats = await fs.stat(this.path);
+    if (await fs.pathExists(this.path) && stats.isDirectory()) {
+    
+        const ocflVersion = await this.isObject(this.path);
+        if (!ocflVersion) {
+          throw new Error('can\'t initialise an object here as there are already files')
+        }
+    } else {
+      throw new Error(path + ' does not exist or is not a directory');
+    }
+  }
+
+  getVersionString(i) {
+    // Make a version name as per the SHOULD in the spec v1..vn
+    // TODO have an option for zero padding
+    return "v" + i
+  }
+
+  async getInventory() {
+    const inventoryPath = path.join(this.path, "inventory.json");
+    if (await fs.exists(inventoryPath)) {
+      return await JSON.parse(fs.readFileSync(inventoryPath));
+    }
+    else {
+      return null;
+    }
+  }
+
+  async determineVersion() {
+    const inv = await this.getInventory();
+    if (inv) {
+      return inv.head;
+    }
+    else {
+      return null;
+    }
+    // Here's not how to do it: 
+    /* var version = 0;
+    const dirContents = await fs.readdir(this.path);
+    for (let f of dirContents.filter(function(d){return d.match(/^v\d+$/)})){    
+        const v =  parseInt(f.replace("v",""));
+        if (v > version) {
+            version = v;
+        }
+    }
+    return version;10. */
+    // Look at each dir that matches v\d+
+
+  }
+
+  async isObject(aPath) {
+    // TODO: Check if this content root with NAMASTE and returns ocfl version
+    // 0=ocfl_object_1.0
+    // looks at path and see if the content of the file is
+    // TODO: Make this look for a namaste file beginning with 0=ocfl_object_ and extract the version
+    const theFile = path.join(aPath, "0=" + this.nameVersion(this.ocflVersion));
+    return await fs.pathExists(theFile);
+  }
+
+  nameVersion(version) {
+    return 'ocfl_object_' + version;
+  }
+
+  async generateNamaste(aPath, version) {
+    const fileName = '0=' + this.nameVersion(version);
+    const thePath = path.join(aPath, fileName);
+    const writeFile = await fs.writeFile(thePath, this.nameVersion(version));
+    const contentDir = await fs.mkdir(path.join(aPath, "v1"));
+  }
+
+
+  async digest_dir(dir) {
+    var items = {};
+    const contents = await fs.readdir(dir);
+    items = _.flatten(await Promise.all(contents.map(async (p1) => {
+      const p = path.join(dir, p1);
+      const stats = await fs.stat(p);
+      if (stats.isDirectory()) {
+        return await this.digest_dir(p);
+      } else {
+        const h = await this.hash_file(p);
+        return [p, h];
+      }
+    })));
+    return items;
+  }
+
+  async hash_file(p) {
+    const hash = await hasha.fromFile(p, { algorithm: DIGEST_ALGORITHM })
+    return hash;
+  }
+
+
+  async removeEmptyDirectories(folder) {
+    // Remove empty directories
+    // Adapted (nade async) from: https://gist.github.com/jakub-g/5903dc7e4028133704a4
+    if (!folder) {
+      folder = this.path;
+    }
+    const stats = await fs.stat(folder);
+    var isDir = await stats.isDirectory();
+    if (isDir) {
+      var files = await fs.readdir(folder);
+      if (files.length > 0) {
+        for (let f of files) {
+          var fullPath = path.join(folder, f);
+          await this.removeEmptyDirectories(fullPath);
+        }
+        files = await fs.readdir(folder);
+      }
+      if (files.length == 0) {
+        const rm = await fs.rmdir(folder);
+        return;
+      }
+    }
+  }
+
+  async inventory(id, dir) {
+    const versionId = "v1";
+    const inv = {
+      'id': id,
+      'type': 'https://ocfl.io/1.0/spec/#inventory',
+      'digestAlgorithm': DIGEST_ALGORITHM,
+      'head': versionId,
+      'versions': {
+
+      }
+    };
+    inv.versions[versionId] = {
+      "created": new Date().toISOString(),
+      "state": {}
+    }
+    // TODO Message and state keys in version
+    var hashpairs = await this.digest_dir(dir);
+    var versionState = inv.versions[versionId].state;
+    inv['manifest'] = {};
+    for (let i = 0; i < hashpairs.length; i += 2) {
+      const thisHash = hashpairs[i + 1]
+      const thisPath = path.relative(this.path, hashpairs[i])
+      const versionPath = path.relative(path.join("v1", "content"), thisPath)
+      if (!inv['manifest'][thisHash]) {
+        //Store only ONE path
+        inv['manifest'][thisHash] = [thisPath]
+      }
+      else {
+        // TODO DELETE THIS FILE FROM THE OBJECT BEFORE WE STORE IT
+      }
+
+      if (versionState[thisHash]) {
+        // Store all 
+        versionState[thisHash].push(versionPath)
+      }
+      else {
+        versionState[thisHash] = [versionPath]
+
+      }
+
+    }
+    return inv
+  }
+
+  async getFilePath(filePath, version) {
+    const inv = await this.getInventory();
+    if (!version) {
+      version = inv.head;
+    }
+    const state = inv.versions[version].state;
+    for (let hash of Object.keys(state)) {
+      const aPath = state[hash];
+      const findPath = _.find(aPath, (p) => p === filePath);
+      if (findPath) {
+        return inv.manifest[hash][0];
+      }
+    }
+    throw new Error('cannot find file');
+  }
+
+}

package/lib/transaction.js

@@ -0,0 +1,413 @@
+//@ts-check
+/** Models a set of ocfl object modifying operations as a transaction that is as close as atomic as possible */
+const { createReadStream } = require('fs');
+const { stat, readdir } = require('fs/promises');
+const path = require('path');
+const { parallelize, dataSourceAsIterable } = require('./utils');
+const { OcflDigest } = require('./digest');
+const { INVENTORY_NAME } = require('./constants').OcflConstants;
+
+//const { pipeline } = require('stream/promises');
+//const { OcflObjectImpl, OcflObjectInventory } = require('./object');
+
+/**
+ * @typedef {import('./object').OcflObjectImpl} OcflObjectImpl
+ * @typedef {import('./store').OcflStore} OcflStore
+ * @typedef {import('./inventory').OcflObjectInventoryMut} OcflObjectInventoryMut
+ * @typedef {import('stream').Writable} Writable
+ * @typedef {import('stream').StreamOptions} StreamOptions
+ */
+
+/**
+ * A Transaction represent an atomic update on an OCFL Object. 
+ * When a transaction is commited, all changes are saved as a new version.
+ * This is a wrapper class to hide all the storage specific implementation.
+ */
+class OcflObjectTransaction {
+
+  /**
+   * Discard all the changes made in this transaction,
+   * @abstract
+   */
+  async rollback() { throw new Error('Not Implemented'); }
+
+  /**
+   * Commit all the changes made in this transaction to the actual OCFL Object.
+   * @abstract
+   * @param {Object} [options] - Commit options. message and user will be included in the version entry of the inventory.json
+   * @param {string} [options.message] - Freeform text used to record the rationale for creating the current version
+   * @param {Object} [options.user] - Identify the user or agent that created the current Version
+   * @param {string} options.user.name - Any readable name of the user, e.g., a proper name, user ID, agent ID.
+   * @param {string} [options.user.address] - A URI: either a mailto URI [RFC6068] with the e-mail address of the user or a URL to a personal identifier, e.g., an ORCID iD.
+   */
+  async commit(options) { throw new Error('Not Implemented'); }
+
+  /**
+   * Completely remove a file path and corresponding content from all versions of an OCFL Object.
+   * This is a special case that is not supported as part of regular OCFL versioning operations.
+   * @param {*} logicalPath 
+   */
+  async purge(logicalPath) {
+    return await this.remove(logicalPath, { purge: true });
+  }
+
+
+  /**
+   * Write data to a file at `logicalPath`
+   * @abstract
+   * @param {string} logicalPath - A path relative to the object content directory
+   * @param {*} data - The same data as https://nodejs.org/api/fs.html#fs_fs_writefile_file_data_options_callback
+   * @param {Object|string} [options] - The same options as https://nodejs.org/api/fs.html#fs_fs_writefile_file_data_options_callback
+   */
+  async write(logicalPath, data, options) { throw new Error('Not Implemented'); }
+
+  /**
+   * Import content by recursively copying a file or directory from local file system to the 
+   * OCFL Object content. For example, `copy('/home/john/data/b','data/b')` will
+   * copy `/home/john/data/b` from the local filesystem to the content `data/b` in the OFCL Object.
+   * Set target to empty string ('') to copy a directory content to the object's content root.
+   * If target is not specified, the base name of the source will be used.
+   * @abstract
+   * @param {string} source - A path to a file or directory in the local filesystem to be copied
+   * @param {string} [target] - A logical path in the OCFL Object content.
+   */
+  async import(source, target) { throw new Error('Not Implemented'); }
+
+  /**
+   * Duplicate the logical path `source` as a different logical path `target`, both paths will then point
+   * to the same content path and bitstreams.
+   * @abstract
+   * @param {string} source - The logical path to be copied
+   * @param {string} target - The target logical path
+   */
+  async copy(source, target) { throw new Error('Not Implemented'); }
+
+  /**
+  * Changes the logical path of existing content. This is similar to move operation in file systems
+  * This operation will NOT rename the actual file name referenced by the manifest.
+  * @abstract
+  * @param {*} source 
+  * @param {*} target 
+  */
+  async rename(source, target) { throw new Error('Not Implemented'); }
+
+  /**
+   * Changes the file path of existing content. This is similar to move operation in file systems
+   * @param {*} source 
+   * @param {*} target 
+   */
+  async move(source, target) { return this.rename(source, target); }
+
+  /**
+  * Remove a file or directory
+  * @abstract
+  * @param {string} logicalPath - A path relative to the object content directory
+  * @param {Object} [options]
+  */
+  async remove(logicalPath, options) { throw new Error('Not Implemented'); }
+
+  /**
+   * Remove a file or directory
+   * @param {string} logicalPath - A path relative to the object content directory
+   * @param {Object} [options]
+   */
+  async delete(logicalPath, options) { return this.remove(logicalPath, options); }
+
+  /**
+   * Makes content from a version earlier than the previous version available in the current version of an OCFL Object.
+   * @abstract
+   * @param {string} logicalPath 
+   * @param {string} versionName 
+   * @return {Promise<number>}
+   */
+  async reinstate(logicalPath, versionName) { throw new Error('Not Implemented'); }
+
+  /**
+   * Create a writable stream
+   * @abstract
+   * @param {string} logicalPath 
+   * @param {BufferEncoding|StreamOptions} [options] 
+   * @return {Promise<import('stream').Writable>} 
+   */
+  async createWriteStream(logicalPath, options) { throw new Error('Not Implemented'); }
+
+}
+
+
+/**
+ * Base class for any transaction implementation on different storage backend
+ */
+class OcflObjectTransactionImpl extends OcflObjectTransaction {
+  _contentRoot;
+  _object;
+  _inventory;
+  _committed = false;
+  _unfinished = 0;
+
+
+  /**
+   * 
+   * @param {OcflObjectImpl} ocflObject 
+   * @param {OcflStore} ocflStore 
+   * @param {OcflObjectInventoryMut} inventory 
+   * @param {string} workspaceVersionPath 
+   * @param {string} createdDir 
+   */
+  constructor(ocflObject, ocflStore, inventory, workspaceVersionPath, createdDir) {
+    super();
+    this._workspaceVersionPath = workspaceVersionPath;
+    this._contentRoot = path.join(workspaceVersionPath, inventory.contentDirectory);
+    this._object = ocflObject;
+    this._store = ocflStore;
+    this._inventory = inventory;
+    this._createdDir = createdDir;
+    this._queue = [];
+  }
+
+  _getRealPath(logicalPath) {
+    let p = path.normalize(logicalPath);
+    if (p.startsWith('/') || p.startsWith('..')) throw new Error('logicalPath must be a relative path without `..`');
+    return path.join(this._contentRoot, p);
+  }
+
+  async rollback() {
+    if (!this._committed) {
+      await Promise.all(this._queue);
+      if (this._createdDir) await this._store.remove(this._createdDir);
+      this._committed = true;
+    }
+  }
+
+  async commit(options) {
+    // console.log('commit');
+    // console.log(this._inventory.toString());
+    if (!this._committed) {
+      let count = this._unfinished;
+      if (count > 0) {
+        await this.rollback();
+        throw new Error(`Unfinished operations detected (${count}). Make sure that updater callback is async function and there are no floating promises inside.`);
+      }
+      // if there is no changes, abort commit
+      if (!this._inventory.isChanged) {
+        return await this.rollback();
+      }
+      await this._commit(options);
+      this._object._setInventory(this._inventory);
+      this._committed = true;
+    }
+
+  }
+
+  async _commit(options = {}) {
+    // set commit info
+    this._inventory.created = (new Date()).toISOString();
+    if (options.message) this._inventory.message = options.message;
+    if (options.user) this._inventory.user = options.user;
+
+    let workspaceVersionPath = this._workspaceVersionPath;
+    let objectVersionPath = path.join(this._object.root, this._inventory.head);
+    /* eg: /workspace/aa/bb/cc/object-1/v1/inventory.json */
+    let invPath = path.join(workspaceVersionPath, INVENTORY_NAME);
+    /* eg: /workspace/aa/bb/cc/object-1/v1/inventory.json.sha512 */
+    let invDigestPath = invPath + '.' + this._inventory.digestAlgorithm;
+
+    // calculate digest
+    let digest = this._inventory.digest() + ' ' + INVENTORY_NAME;
+    // write inventory.json to workspace
+    await this._store.writeFile(invPath, this._inventory.toString());
+    await this._store.writeFile(invDigestPath, digest);
+    if (workspaceVersionPath !== objectVersionPath) {
+      try {
+        await this._object._ensureNamaste();
+        await this._store.move(workspaceVersionPath, objectVersionPath);
+        await this._store.remove(this._createdDir);
+      } catch (error) {
+        await this.rollback();
+        throw error;
+      }
+    }
+    // replace root inventory
+    /* eg: /data/aa/bb/cc/object-1/inventory.json */
+    let rootInvPath = path.join(this._object.root, INVENTORY_NAME);
+    /* eg: /data/aa/bb/cc/object-1/v1/inventory.json */
+    invPath = path.join(objectVersionPath, INVENTORY_NAME);
+    invDigestPath = invPath + '.' + this._inventory.digestAlgorithm;
+    let rootInvDigestPath = rootInvPath + '.' + this._inventory.digestAlgorithm;
+    await this._store.copyFile(invPath, rootInvPath + '.tmp');
+    await Promise.all([
+      this._store.move(rootInvPath + '.tmp', rootInvPath),
+      this._store.copyFile(invDigestPath, rootInvDigestPath)]);
+
+  }
+
+  /**
+   * 
+   * @param {string} logicalPath 
+   * @param {Object} options
+   * @return {Promise<import('stream').Writable>} 
+   */
+  async createWriteStream(logicalPath, options) {
+    let realPath = this._getRealPath(logicalPath);
+    let ws = await this._store.createWriteStream(realPath, options);
+    let hs = OcflDigest.createStream(this._inventory.digestAlgorithm);
+    const wwrite = ws.write;
+    function nwrite(chunk, encoding, cb) {
+      hs.update(chunk, encoding);
+      return wwrite.apply(ws, arguments);
+    };
+    ws.write = nwrite;
+    ws.on('close', async () => {
+      let digest = hs.digest('hex');
+      if (this._inventory.getContentPath(digest)) await this._store.remove(realPath);
+      this._inventory.add(logicalPath, digest);
+    });
+    return ws;
+  }
+
+  async write(logicalPath, data, options) {
+    let realPath = this._getRealPath(logicalPath);
+    let dataSrc = dataSourceAsIterable(data);
+    let digest;
+    if (Array.isArray(dataSrc)) {
+      // if data is not stream, check digest first
+      digest = await OcflDigest.digestAsync(this._inventory.digestAlgorithm, dataSrc);
+      if (!this._inventory.getContentPath(digest)) {
+        // no digest yet, write the file to storage backend
+        await this._store.writeFile(realPath, data, options);
+      }
+    } else {
+      // save the stream first
+      let hs = OcflDigest.createStreamThrough(this._inventory.digestAlgorithm);
+      data.pipe(hs);
+      await this._store.writeFile(realPath, hs, options);
+      digest = hs.digest();
+      // already exists, delete temp file
+      if (this._inventory.getContentPath(digest)) await this._store.remove(realPath);
+    }
+    this._inventory.add(logicalPath, digest);
+    // console.log(this._inventory.toString());
+  }
+
+  async importFile(source, target) {
+    if (!target) throw new TypeError('Target logical path must not be empty if source is a file.');
+    let realPath = this._getRealPath(target);
+    let digest = await OcflDigest.digestFromFile(this._inventory.digestAlgorithm, source);
+
+    if (!this._inventory.getContentPath(digest)) {
+      await this._store.copyFile(source, realPath);
+    }
+    this._inventory.add(target, digest);
+  }
+
+  async import(source, target) {
+    target = target ?? path.basename(source);
+    let srcStat = await stat(source);
+    if (srcStat.isFile()) {
+      await this.importFile(source, target);
+    } else if (srcStat.isDirectory()) {
+      const files = await readdir(source);
+      await parallelize(files, async (filename) => {
+        await this.import(path.join(source, filename), path.join(target, filename));
+      });
+    }
+  }
+
+  async copy(source, target) {
+    return this._inventory.copy(source, target);
+  }
+
+  async rename(source, target) {
+    // if file already exists in the workspace, rename the actual file
+    try {
+      let realSource = this._getRealPath(source);
+      let realTarget = this._getRealPath(target);
+      await this._store.move(realSource, realTarget);
+    } catch (error) {
+      if (error.code !== 'ENOENT') throw error;
+    }
+    return this._inventory.rename(source, target);
+  }
+
+  async reinstate(logicalPath, versionName) {
+    //let digest = this._inventory.getDigestFromLogical(logicalPath);
+    let count = this._inventory.reinstate(logicalPath, versionName);
+    // if (count) {
+    //   // ensure no existing file with the same logical path in the workspace
+    //   try {
+    //     await this._store.remove(this._getRealPath(logicalPath));
+    //   } catch (error) {
+    //     if (error.code !== 'ENOENT') throw error;
+    //   }
+    // }
+    return count;
+  }
+
+  async remove(logicalPath, options) {
+    if (this._inventory.delete(logicalPath)) {
+      try {
+        await this._store.remove(this._getRealPath(logicalPath));
+      } catch (error) {
+      }
+    }
+  }
+
+}
+
+
+const transactionInterface = new OcflObjectTransaction();
+const methodWrapper = {
+  createWriteStream: async function (logicalPath, options) {
+    if (this._committed) throw new Error('Transaction already commited');
+    this._unfinished++;
+    let result = await this.createWriteStream(logicalPath, options);
+    result.on('close', () => --this._unfinished);
+    return result;
+  }
+};
+['write', 'import', 'copy', 'rename', 'remove', 'reinstate'].forEach(name => {
+  methodWrapper[name] = async function () {
+    if (this._committed) throw new Error('Transaction already commited');
+    this._unfinished++;
+    let p = this[name](...arguments);
+    this._queue.push(p);
+    try {
+      let result = await p;
+      this._unfinished--;
+      return result;
+    } catch (error) {
+      this._unfinished--;
+      throw error;
+    }
+  };
+});
+const proxyHandler = {
+  /**
+   * @param {OcflObjectTransactionImpl} target 
+   * @param {string} prop 
+   */
+  get(target, prop) {
+    if (prop in transactionInterface) {
+      let p = target[prop];
+      if (typeof p === 'function') {
+        return methodWrapper[prop]?.bind(target) ?? p.bind(target);
+      } else {
+        return p;
+      }
+    }
+  }
+}
+
+/**
+ * Return any OcflObjectTransactionImpl implementation as OcflObjectTransaction.
+ * @param {OcflObjectTransactionImpl} impl
+ * @return {OcflObjectTransaction}
+ */
+function createTransactionProxy(impl) {
+  return new Proxy(impl, proxyHandler);
+}
+
+module.exports = {
+  createTransactionProxy,
+  OcflObjectTransaction,
+  OcflObjectTransactionImpl
+};