@ocfl/ocfl-fs 0.2.0 → 0.3.0

package/index.js

@@ -1,10 +1,14 @@
 //@ts-check
 const { OcflFsStore } = require('./lib/store.js');
-const { Ocfl } = require('@ocfl/ocfl');
+const { implementOcfl } = require('@ocfl/ocfl');
 //const { default: ocfl } = require('./index.mjs');
 
 /** @type {import('./lib/store.js').OcflFsStoreConfig} */
 let defaultOptions;
 
-module.exports = new Ocfl(OcflFsStore, defaultOptions);
+const ocfl = implementOcfl(OcflFsStore, defaultOptions);
+const { OCFL_VERSIONS, StorageLayout, createStorage, loadStorage, object, storage, storageLayout } = ocfl;
+module.exports = {
+  OCFL_VERSIONS, StorageLayout, createStorage, loadStorage, object, storage, storageLayout 
+};
 //module.exports.default = module.exports;

package/index.mjs

@@ -1,3 +1,4 @@
 export * from './index.js'; 
-import ocfl from './index.js';
+import * as ocfl from './index.js';
 export default ocfl;
+

package/lib/store.js

@@ -1,9 +1,10 @@
 //@ts-check
 const path = require('path');
-const fs = require('fs');
-const { mkdir, copyFile, rename, unlink } = fs.promises;
+const fs = require('node:fs');
+const { Readable, Writable } = require('node:stream');
 
 const { OcflStore, OcflConstants } = require('@ocfl/ocfl');
+const { opendir } = require('./utils.js');
 const { INVENTORY_NAME } = OcflConstants;
 
 /**
@@ -37,8 +38,8 @@ class OcflFsStore extends OcflStore {
     this.fs = config?.fs ?? fs;
   }
 
-  async stat(filePath) { 
-    return this.fs.promises.stat(filePath);    
+  async stat(filePath) {
+    return this.fs.promises.stat(filePath);
   }
 
   async createReadStream(filePath, options) {
@@ -46,11 +47,20 @@ class OcflFsStore extends OcflStore {
     return this.fs.createReadStream(filePath, options);
   }
 
-  async createWriteStream(filePath, options) { 
+  async createWriteStream(filePath, options) {
     await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
     return this.fs.createWriteStream(filePath, options);
   }
 
+  async createReadable(filePath, options) {
+    return /** @type {ReadableStream} */ (Readable.toWeb(this.fs.createReadStream(filePath, options)));
+  }
+
+  async createWritable(filePath, options) {
+    await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+    return Writable.toWeb(this.fs.createWriteStream(filePath, options));
+  }
+
   async readFile(filePath, options) {
     return this.fs.promises.readFile(filePath, options);
     //if (err.code !== 'ENOENT') throw validation.createError(34, invPath);
@@ -58,6 +68,7 @@ class OcflFsStore extends OcflStore {
 
   async writeFile(filePath, data, options) {
     await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+    if (data instanceof ReadableStream) data = fs.ReadStream.fromWeb(data);
     await this.fs.promises.writeFile(filePath, data, options);
   }
 
@@ -67,11 +78,54 @@ class OcflFsStore extends OcflStore {
   }
 
   async opendir(filePath, options) {
-    return this.fs.promises.opendir(filePath, options); 
+    return this.fs.promises.opendir(filePath, options);
   }
 
   async readdir(filePath, options) {
-    return this.fs.promises.readdir(filePath, options); 
+    return this.fs.promises.readdir(filePath, options);
+  }
+
+  /**
+   * @param {string} dirPath 
+   * @param {OpenDirOptions} [options]
+   * @return {Promise<AsyncIterableIterator<{name: string, path: string, size: number, lastModified: Date}>> }
+   */
+  async list(dirPath, { encoding = 'utf8', bufferSize = 32, recursive = false } = {}) {
+    const {opendir, stat} = this.fs.promises;
+    const maxBuffer = bufferSize || 1;
+    async function* generator() {
+      /** @type Promise<{ name: string, path: string, size: number, lastModified: Date}>[] */
+      let buffer = [];
+      let dirp;
+      const dirQueue = [opendir(dirPath, { encoding, bufferSize })];
+      while (dirp = dirQueue.shift()) {
+        const dir = await dirp;
+        for await (const de of dir) {
+          const cdp = path.join(dir.path, de.name);
+          if (recursive && de.isDirectory()) {
+            dirQueue.push(opendir(cdp, { encoding, bufferSize }));
+          } else {
+            let f = stat(cdp).then(stats => ({
+              name: de.name,
+              path: path.relative(dirPath, cdp),
+              size: stats.size,
+              lastModified: stats.mtime
+            }));
+            if (buffer.length < maxBuffer) {
+              buffer.push(f);
+            } else {
+              yield buffer.shift();
+              buffer.push(f);
+            }
+          }
+        }
+      }
+      let file;
+      while (file = buffer.shift()) {
+        yield file;
+      }
+    }
+    return generator();
   }
 
   async mkdir(filePath, options = { recursive: true }) {
@@ -80,6 +134,7 @@ class OcflFsStore extends OcflStore {
 
   async move(source, target) {
     try {
+      await this.fs.promises.mkdir(path.dirname(target), { recursive: true });
       return this.fs.promises.rename(source, target);
     } catch (error) {
       if (error.code === 'EXDEV') {

package/lib/types.d.ts

@@ -1,3 +1,5 @@
-type TestType = {
-  name: string;
-};
+interface OpenDirOptions {
+  encoding?: BufferEncoding | undefined;
+  bufferSize?: number | undefined;
+  recursive?: boolean | undefined;
+}

package/lib/utils.js

@@ -0,0 +1,62 @@
+const fs = require('node:fs/promises');
+const path = require('node:path');
+
+/** 
+ * Wrapper for fs.opendir that reimplements the recursion and 
+ * returns just the file without the directory itself
+ * fs.opendir has a bug that gives wrong result when recursive option is used
+ * @param {string} dirPath
+ * @param {OpenDirOptions} [options]
+ */
+async function* opendir(dirPath, { encoding = 'utf8', bufferSize = 32, recursive = false } = {}) {
+  /** @type Promise<{ name: string, path: string, size: number, lastModified: Date}>[] */
+  let buffer = [];
+  let dirp;
+  const maxBuffer = bufferSize || 1;
+  const dirQueue = [fs.opendir(dirPath, { encoding, bufferSize })];
+  while (dirp = dirQueue.shift()) {
+    const dir = await dirp;
+    for await (const de of dir) {
+      const cdp = path.join(dir.path, de.name);
+      if (recursive && de.isDirectory()) {
+        dirQueue.push(fs.opendir(cdp));
+      } else {
+        let f = fs.stat(cdp).then(stats => ({
+          name: de.name,
+          path: path.relative(dirPath, cdp),
+          size: stats.size,
+          lastModified: stats.mtime
+        }));
+        if (buffer.length < maxBuffer) {
+          buffer.push(f);
+        } else {
+          yield buffer.shift();
+          buffer.push(f);
+        }
+      }
+    }
+  }
+  let file;
+  while (file = buffer.shift()) {
+    yield file;
+  }
+}
+
+// (async function() {
+  // var a = opendir('/home/alvinsw/.ssh', {recursive: true})
+  // var b = 'a';
+  // console.log(await a);
+
+  // let c = 0;
+  // for await (const f of await opendir('/home/alvinsw/uqlang', {recursive: true})) {
+  //   c++;
+  // }
+  // console.log(c);
+  // c=0;
+  // for await (const f of await fs.opendir('/home/alvinsw/uqlang', {recursive: true})) {
+  //   c++;
+  // }
+  // console.log(c);
+// })();
+
+module.exports = { opendir }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ocfl/ocfl-fs",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Oxford Common File Layout (OCFL) JS client library for filesystem storage",
   "main": "index.js",
   "exports": {
@@ -15,8 +15,7 @@
   },
   "scripts": {
     "test": "mocha",
-    "build-docs2": "typedoc --tsconfig tsconfig.json",
-    "build-docs": "jsdoc -c jsdoc.json"
+    "build-docs": "typedoc --plugin typedoc-plugin-missing-exports"
   },
   "keywords": [
     "ocfl",
@@ -36,15 +35,17 @@
   ],
   "license": "GPL-3.0-or-later",
   "devDependencies": {
-    "@types/fs-extra": "^9.0.13",
-    "clean-jsdoc-theme": "^3.3.4",
-    "jsdoc": "^3.6.10",
-    "mocha": "^10.0.0",
-    "typedoc": "^0.22.15",
-    "@ocfl/ocfl-tests": "*"
+    "@ocfl/ocfl-tests": "*",
+    "@types/fs-extra": "^11.0.4",
+    "@types/node": "^24",
+    "clean-jsdoc-theme": "^4.3.0",
+    "jsdoc": "^4.0.3",
+    "mocha": "^11.7.5",
+    "typedoc": "^0.28.14",
+    "typedoc-plugin-missing-exports": "^4.1.2"
   },
   "dependencies": {
-    "@ocfl/ocfl": "0.2.0"
+    "@ocfl/ocfl": "0.3.0"
   },
   "files": [
     "index.js",