@ocfl/ocfl-fs 0.1.2 → 0.2.2

package/README.md

@@ -1,4 +1,22 @@
-# ocfl-js - Filesystem storage implementation
+# ocfl-fs - OCFL implementation using Filesystem storage backend for Node.js
 This is JavaScript/Node.js library to create and interact with Oxford Common File Layout (OCFL) storage and objects within it. 
 This package implements the OCFL client that stores storage root and its objects on a locally attached filesystem.
 
+## Installation
+
+    npm install @ocfl/ocfl-fs
+
+## Usage
+
+The storage and object methods accept an optional fs parameter, which can be any module that implements Node.js fs module API. By default, it will use the built in fs module.
+
+    const ocfl = require('@ocfl/ocfl-fs');
+    const storage = ocfl.storage({root: '/var/data/myocfl'});
+
+For example, by passing it memfs (https://www.npmjs.com/package/memfs) the OCFL storage will be created in memory using the memfs module: 
+
+    const ocfl = require('@ocfl/ocfl-fs');
+    const fs = require('memfs');
+    const storage = ocfl.storage({root: '/var/data/myocfl'}, {fs});
+
+For common usage documentation please refer to the [README.md in GitHub](https://github.com/Arkisto-Platform/ocfl-js)

package/lib/store.js

@@ -1,9 +1,10 @@
 //@ts-check
 const path = require('path');
-const fs = require('fs');
-const { mkdir, copyFile, rename, unlink } = fs.promises;
+const fs = require('node:fs');
+const { Readable, Writable } = require('node:stream');
 
 const { OcflStore, OcflConstants } = require('@ocfl/ocfl');
+const { opendir } = require('./utils.js');
 const { INVENTORY_NAME } = OcflConstants;
 
 /**
@@ -18,11 +19,6 @@ class OcflFsStore extends OcflStore {
   /** @type {Map<Object, OcflFsStore>} */
   static instances = new Map();
 
-  /**
-   * 
-   * @param {Object} options 
-   * @param {typeof fs} [options.fs]
-   */
   // static getInstance({fs = require('fs')} = {}) {
   //   let store = OcflFsStore.instances.get(fs);
   //   if (!store) {
@@ -42,8 +38,8 @@ class OcflFsStore extends OcflStore {
     this.fs = config?.fs ?? fs;
   }
 
-  async stat(filePath) { 
-    return this.fs.promises.stat(filePath);    
+  async stat(filePath) {
+    return this.fs.promises.stat(filePath);
   }
 
   async createReadStream(filePath, options) {
@@ -51,11 +47,20 @@ class OcflFsStore extends OcflStore {
     return this.fs.createReadStream(filePath, options);
   }
 
-  async createWriteStream(filePath, options) { 
+  async createWriteStream(filePath, options) {
     await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
     return this.fs.createWriteStream(filePath, options);
   }
 
+  async createReadable(filePath, options) {
+    return Readable.toWeb(this.fs.createReadStream(filePath, options));
+  }
+
+  async createWritable(filePath, options) {
+    await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+    return Writable.toWeb(this.fs.createWriteStream(filePath, options));
+  }
+
   async readFile(filePath, options) {
     return this.fs.promises.readFile(filePath, options);
     //if (err.code !== 'ENOENT') throw validation.createError(34, invPath);
@@ -72,11 +77,54 @@ class OcflFsStore extends OcflStore {
   }
 
   async opendir(filePath, options) {
-    return this.fs.promises.opendir(filePath, options); 
+    return this.fs.promises.opendir(filePath, options);
   }
 
   async readdir(filePath, options) {
-    return this.fs.promises.readdir(filePath, options); 
+    return this.fs.promises.readdir(filePath, options);
+  }
+
+  /**
+   * @param {string} dirPath 
+   * @param {OpenDirOptions} [options]
+   * @return {Promise<AsyncIterableIterator<{name: string, path: string, size: number, lastModified: Date}>> }
+   */
+  async list(dirPath, { encoding = 'utf8', bufferSize = 32, recursive = false } = {}) {
+    const {opendir, stat} = this.fs.promises;
+    const maxBuffer = bufferSize || 1;
+    async function* generator() {
+      /** @type Promise<{ name: string, path: string, size: number, lastModified: Date}>[] */
+      let buffer = [];
+      let dirp;
+      const dirQueue = [opendir(dirPath, { encoding, bufferSize })];
+      while (dirp = dirQueue.shift()) {
+        const dir = await dirp;
+        for await (const de of dir) {
+          const cdp = path.join(dir.path, de.name);
+          if (recursive && de.isDirectory()) {
+            dirQueue.push(opendir(cdp, { encoding, bufferSize }));
+          } else {
+            let f = stat(cdp).then(stats => ({
+              name: de.name,
+              path: path.relative(dirPath, cdp),
+              size: stats.size,
+              lastModified: stats.mtime
+            }));
+            if (buffer.length < maxBuffer) {
+              buffer.push(f);
+            } else {
+              yield buffer.shift();
+              buffer.push(f);
+            }
+          }
+        }
+      }
+      let file;
+      while (file = buffer.shift()) {
+        yield file;
+      }
+    }
+    return generator();
   }
 
   async mkdir(filePath, options = { recursive: true }) {

package/lib/types.d.ts

@@ -1,3 +1,5 @@
-type TestType = {
-  name: string;
-};
+interface OpenDirOptions {
+  encoding?: BufferEncoding | undefined;
+  bufferSize?: number | undefined;
+  recursive?: boolean | undefined;
+}

package/lib/utils.js

@@ -0,0 +1,62 @@
+const fs = require('node:fs/promises');
+const path = require('node:path');
+
+/** 
+ * Wrapper for fs.opendir that reimplements the recursion and 
+ * returns just the file without the directory itself
+ * fs.opendir has a bug that gives wrong result when recursive option is used
+ * @param {string} dirPath
+ * @param {OpenDirOptions} [options]
+ */
+async function* opendir(dirPath, { encoding = 'utf8', bufferSize = 32, recursive = false } = {}) {
+  /** @type Promise<{ name: string, path: string, size: number, lastModified: Date}>[] */
+  let buffer = [];
+  let dirp;
+  const maxBuffer = bufferSize || 1;
+  const dirQueue = [fs.opendir(dirPath, { encoding, bufferSize })];
+  while (dirp = dirQueue.shift()) {
+    const dir = await dirp;
+    for await (const de of dir) {
+      const cdp = path.join(dir.path, de.name);
+      if (recursive && de.isDirectory()) {
+        dirQueue.push(fs.opendir(cdp));
+      } else {
+        let f = fs.stat(cdp).then(stats => ({
+          name: de.name,
+          path: path.relative(dirPath, cdp),
+          size: stats.size,
+          lastModified: stats.mtime
+        }));
+        if (buffer.length < maxBuffer) {
+          buffer.push(f);
+        } else {
+          yield buffer.shift();
+          buffer.push(f);
+        }
+      }
+    }
+  }
+  let file;
+  while (file = buffer.shift()) {
+    yield file;
+  }
+}
+
+// (async function() {
+  // var a = opendir('/home/alvinsw/.ssh', {recursive: true})
+  // var b = 'a';
+  // console.log(await a);
+
+  // let c = 0;
+  // for await (const f of await opendir('/home/alvinsw/uqlang', {recursive: true})) {
+  //   c++;
+  // }
+  // console.log(c);
+  // c=0;
+  // for await (const f of await fs.opendir('/home/alvinsw/uqlang', {recursive: true})) {
+  //   c++;
+  // }
+  // console.log(c);
+// })();
+
+module.exports = { opendir }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ocfl/ocfl-fs",
-  "version": "0.1.2",
+  "version": "0.2.2",
   "description": "Oxford Common File Layout (OCFL) JS client library for filesystem storage",
   "main": "index.js",
   "exports": {
@@ -37,14 +37,19 @@
   "license": "GPL-3.0-or-later",
   "devDependencies": {
     "@types/fs-extra": "^9.0.13",
-    "clean-jsdoc-theme": "^3.3.4",
-    "jsdoc": "^3.6.10",
-    "mocha": "^10.0.0",
+    "@types/node": "^18",
+    "clean-jsdoc-theme": "^4.3.0",
+    "jsdoc": "^4.0.3",
+    "mocha": "^10.7.3",
     "typedoc": "^0.22.15",
-    "@ocfl/ocfl-tests": "0.1.*"
+    "@ocfl/ocfl-tests": "*"
   },
   "dependencies": {
-    "@ocfl/ocfl": "0.1.1"
+    "@ocfl/ocfl": "0.2.2"
   },
-  "files": [ "index.js", "index.mjs", "lib" ]
+  "files": [
+    "index.js",
+    "index.mjs",
+    "lib"
+  ]
 }