@ocfl/ocfl 0.4.0 → 1.0.0-beta.1

package/src/storage.ts

@@ -0,0 +1,342 @@
+//@todo add default layout
+import path from 'node:path';
+import { OcflConstants } from './constants.ts';
+import { NotImplementedError } from './error.ts';
+import { createByName, OcflStorageLayout } from './extension.ts';
+import { HashedNTupleStorageLayout } from './extensions/0004-hashed-n-tuple-storage-layout.ts';
+import { OcflObject } from './object.ts';
+import { findNamasteVersion, isDirEmpty } from './utils.ts';
+import { createError } from './validation.ts';
+
+const { OCFL_VERSION, OCFL_LAYOUT, EXTENSIONS_DIR, EXTENSION_CONFIG, NAMASTE_PREFIX_STORAGE, NAMASTE_PREFIX_OBJECT, NAMASTE_T } = OcflConstants;
+
+import { OcflDigest } from './digest.ts';
+
+const DEFAULT_LAYOUT = HashedNTupleStorageLayout;
+
+import type { OcflStore } from './store.ts';
+import type { OcflExtensionConfig, OcflObjectConfig, OcflStorageConfig, OcflVersion } from './types.ts';
+
+/**
+ * An abstraction to represent the OCFL Storage Root, which is
+ * a base directory used to store OCFL Objects, identified by a [NAMASTE] file “0=ocfl_1.1”.
+ */
+
+class OcflStorage {
+  ocflVersion: OcflVersion = OCFL_VERSION;
+
+  /**
+   * Get the storage root
+   */
+  get root(): string {
+    throw new NotImplementedError();
+  }
+
+  /**
+   * Construct the path of the object root relative to the storage root
+   * The path is mapped from the object identifier using the algorithm defined in the chosen
+   * storage layout extension (specified in {@link OcflStorageConfig}).
+   * @param id - Identifier of the object
+   */
+  objectRoot(_id: string): string {
+    throw new NotImplementedError();
+  }
+
+  /**
+   * Absolute path to the object root
+   * @param id - Identifier of the object
+   */
+  objectRootAbs(id: string) {
+    return path.join(this.root, this.objectRoot(id));
+  }
+
+  /**
+   * Get an existing object in the Storage or create a new one.
+   * @param opt - A unique identifier for the OCFL Object, should be a URI
+   */
+  object(_opt: string | { id?: string | undefined; root?: string | undefined }): OcflObject {
+    throw new NotImplementedError();
+  }
+
+  /**
+   * Check if the storage has already the object with the given id.
+   */
+  async has(id: string) {
+    return this.object(id).exists();
+  }
+
+  /**
+   * Create the storage root in the underlying storage backend
+   * @param options - Additional options for creating storage root
+   */
+  async create(_options?: { ocflSpecText?: boolean | undefined }): Promise<OcflStorage> {
+    throw new NotImplementedError();
+  }
+
+  /**
+   * Check namaste, retrieve storage layout extension, and read its config file.
+   */
+  async load(): Promise<OcflStorage> {
+    throw new NotImplementedError();
+  }
+
+  /**
+   * Check if the storage root path points to an existing file or non-empty directory in the underlying backend store.
+   * The existing directory may or may not be a valid OCFL Storage.
+   */
+  async exists(): Promise<boolean> {
+    throw new NotImplementedError();
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<OcflObject> {
+    throw new NotImplementedError();
+  }
+
+  /**
+   * Return a new AsyncIterator that contains all the OCFL objects in this Storage.
+   */
+  objects(): AsyncIterableIterator<OcflObject> {
+    return this[Symbol.asyncIterator]();
+    // const objects = [];
+    // for await (const o of this) objects.push(o);
+    // return objects;
+  }
+
+  async delete(id: string) {
+    return this.remove(id);
+  }
+
+  async purge(id: string) {
+    return this.remove(id);
+  }
+
+  async remove(_id: string): Promise<void> {
+    throw new NotImplementedError();
+  }
+}
+
+/**
+ * General implementation of {@link OcflStorage} that can uses different datastore backends.
+ * This class provides common functionalities for the subclasses and
+ * at the same provide encapsulation emulating private and protected methods.
+ */
+class OcflStorageImpl extends OcflStorage {
+  #root: string;
+  #workspace: string;
+  #layout: OcflStorageLayout | undefined;
+  #store: OcflStore;
+  #objectConfig: Omit<OcflObjectConfig, 'root' | 'workspace' | 'id' | 'ocflVersion'>;
+
+  constructor(config: OcflStorageConfig, store: OcflStore) {
+    super();
+    if (!store) {
+      throw new TypeError('[OcflStorage] store is required.');
+    }
+    if (!config.root) {
+      throw new TypeError('[OcflStorage] config.root is required.');
+    }
+    this.#store = store;
+    const { root, workspace, layout, ocflVersion, ...objectConfig } = config;
+    this.#root = path.resolve(root);
+    this.#workspace = workspace ? path.resolve(workspace) : path.join(this.#root, 'extensions/workspace');
+    if (this.#workspace?.startsWith(this.#root + path.sep) && !this.#workspace.startsWith(`${this.#root + path.sep}extensions${path.sep}`)) {
+      throw new Error('[OcflStorage] config.workspace cannot be the same as or a subpath of config.root');
+    }
+    if (layout) {
+      this.#layout = layout instanceof OcflStorageLayout ? layout : this.#createLayout(layout);
+    }
+    if (ocflVersion) {
+      this.ocflVersion = ocflVersion;
+    }
+    this.#objectConfig = objectConfig;
+  }
+
+  #createLayout(option: OcflExtensionConfig | string | undefined) {
+    const name = typeof option === 'string' ? option : option?.extensionName;
+    const config = typeof option === 'object' ? option : undefined;
+    if (name) {
+      return createByName<OcflStorageLayout>(OcflStorageLayout, 'Layout', name, config);
+    }
+    // else if (useDefault) {
+    //   return DEFAULT_LAYOUT.create(config);
+    // }
+  }
+
+  get root() {
+    return this.#root;
+  }
+
+  objectRoot(id: string) {
+    if (!this.#layout) {
+      throw new Error('[OcflStorage] No layout defined');
+    }
+    return this.#layout.map(id);
+  }
+
+  /**
+   * The root param is a relative path to the storage root.
+   */
+  object(opt: string | { id?: string | undefined; root?: string | undefined }) {
+    let id: string | undefined;
+    let root: string | undefined;
+    if (typeof opt === 'string') {
+      id = opt;
+    } else {
+      ({ id, root } = opt);
+    }
+    const relObjectRoot = root || (id && this.objectRoot(id));
+    if (!relObjectRoot) {
+      throw new TypeError('[OcflStorage] object id or root is required.');
+    }
+    return new OcflObject(
+      {
+        id,
+        root: path.join(this.root, relObjectRoot),
+        // @todo:use extension workspace if not defined
+        workspace: this.#workspace ? path.join(this.#workspace, relObjectRoot) : undefined,
+        ocflVersion: this.ocflVersion,
+        ...this.#objectConfig,
+      },
+      this.#store,
+    );
+  }
+
+  async exists() {
+    return !(await isDirEmpty(this.#store, this.root));
+  }
+
+  async load() {
+    const prefix = NAMASTE_PREFIX_STORAGE;
+    const [ocflVersion, layout] = (
+      await Promise.allSettled([findNamasteVersion(this.#store, prefix, this.root), this.#store.readFile(path.join(this.root, OCFL_LAYOUT), 'utf8')])
+    ).map((r) => (r.status === 'fulfilled' ? r.value : undefined));
+    if (ocflVersion === '') {
+      throw createError('E080', this.root);
+    }
+    if (!ocflVersion) {
+      throw createError('E069', this.root);
+    }
+    let layoutName: string | undefined;
+    let layoutConfig: OcflExtensionConfig | undefined;
+    // load layout
+    try {
+      if (layout) {
+        layoutName = JSON.parse(layout).extension;
+      }
+    } catch (_error) {}
+    // load ext config if exists
+    if (layoutName) {
+      try {
+        const cfgstr = await this.#store.readFile(path.join(this.root, EXTENSIONS_DIR, layoutName, EXTENSION_CONFIG), 'utf8');
+        layoutConfig = JSON.parse(cfgstr as string);
+      } catch (_error) {}
+    }
+    this.#layout = this.#createLayout(layoutConfig || layoutName);
+    const layoutDigestAlgorithm = this.#layout?.get('digestAlgorithm');
+    if (layoutDigestAlgorithm) {
+      await OcflDigest.initSync(layoutDigestAlgorithm);
+    }
+    //@todo: load storage level extensions
+
+    return this;
+  }
+
+  /**
+   * @param options - Additional options for creating storage root
+   */
+  async create(options?: { ocflSpecText?: boolean | undefined }) {
+    if (await this.exists()) {
+      //cannot create, storage root already exists .
+      throw new Error('[OcflStorage] Cannot create storage root in a non-empty directory');
+    } else {
+      await this.#store.mkdir(this.root);
+    }
+    // create namaste
+    const prefix = NAMASTE_PREFIX_STORAGE;
+    const filenamePrefix = NAMASTE_T + prefix;
+    const filePath = path.join(this.root, filenamePrefix + this.ocflVersion);
+    await this.#store.writeFile(filePath, `${prefix + this.ocflVersion}\n`, 'utf8');
+
+    // create layout
+    if (!this.#layout) {
+      this.#layout = new DEFAULT_LAYOUT();
+    }
+    const layoutDigestAlgorithm = this.#layout.get('digestAlgorithm');
+    if (layoutDigestAlgorithm) {
+      await OcflDigest.initSync(layoutDigestAlgorithm);
+    }
+    const layout = { extension: this.#layout.name, description: this.#layout.description };
+    await this.#store.writeFile(path.join(this.root, OCFL_LAYOUT), JSON.stringify(layout, null, 2), 'utf8');
+    if (this.#layout.config) {
+      await this.#store.writeFile(
+        path.join(this.root, EXTENSIONS_DIR, this.#layout.name, EXTENSION_CONFIG),
+        JSON.stringify(this.#layout.config, null, 2),
+        'utf8',
+      );
+    }
+
+    // todo: create other extensions
+    if (options?.ocflSpecText) {
+    }
+    return this;
+  }
+
+  /**
+   * Allow async iteration over all OCFL objects in this storage.
+   */
+  [Symbol.asyncIterator](): AsyncIterableIterator<OcflObject> {
+    const store = this.#store;
+    const workspace = this.#workspace;
+    const config = this.#objectConfig;
+    const root = this.root;
+    const stack = [this.#store.opendir(this.root)];
+    return {
+      [Symbol.asyncIterator]() {
+        return this;
+      },
+      async next(): Promise<IteratorResult<OcflObject>> {
+        for (let dirp = stack.pop(); dirp; dirp = stack.pop()) {
+          const dir = await dirp;
+          const dirent = await dir.read();
+          if (!dirent) {
+            await dir.close();
+            continue;
+          }
+          stack.push(dirp);
+          if (!dirent.isDirectory()) {
+            continue;
+          }
+          const basePath = path.join(dir.path, dirent.name);
+          let nv: OcflVersion | '' | undefined;
+          try {
+            nv = await findNamasteVersion(store, NAMASTE_PREFIX_OBJECT, basePath);
+          } catch (_err) {}
+          if (nv) {
+            //ocfl object exists
+            // inventory exists
+            const object = new OcflObject(
+              {
+                root: basePath,
+                workspace: workspace ? path.join(workspace, path.relative(root, basePath)) : undefined,
+                ocflVersion: nv,
+                ...config,
+              },
+              store,
+            );
+            //await object.load();
+            return { done: false, value: object };
+          } else {
+            //ocfl object does not exist, process subdir
+            stack.push(store.opendir(basePath));
+          }
+        }
+        return { done: true, value: null };
+      },
+      // return() {
+      //   return { done: true };
+      // }
+    };
+  }
+}
+
+export { OcflStorage, OcflStorageImpl };

package/src/store.ts

@@ -0,0 +1,199 @@
+import type { Dir, ReadStream, Stats, WriteStream } from 'node:fs';
+import path from 'node:path';
+import { joinTypedArray } from './utils.ts';
+
+const emptyOptions = {};
+
+type Encoding = BufferEncoding | { encoding: BufferEncoding };
+
+export type WriteFileData = string | Uint8Array | DataView | ArrayBuffer | ReadableStream;
+
+export interface ListedFile {
+  name: string;
+  path: string;
+  size: number;
+  lastModified: Date;
+}
+
+export interface ListOptions {
+  encoding?: BufferEncoding;
+  bufferSize?: number;
+  recursive?: boolean;
+}
+
+/**
+ * Abstract class to provide common APIs to access and modify the actual data store backend
+ * such as local file system and cloud storage
+ */
+class OcflStore {
+  static instances: Map<object, OcflStore>;
+
+  static getInstance<T extends OcflStore>(this: new (options?: object) => T, options: object = emptyOptions): T {
+    // biome-ignore lint/complexity/noThisInStatic: reads the subclass's own instances map
+    const instances = (this as unknown as typeof OcflStore).instances as Map<object, T> | undefined;
+    if (!instances) {
+      return new this(options);
+    }
+    let store = instances.get(options);
+    if (!store) {
+      store = new this(options);
+      instances.set(options, store);
+    }
+    return store;
+  }
+
+  // biome-ignore lint/complexity/noUselessConstructor: documents the constructor signature subclasses receive from getInstance()
+  constructor(_options?: object) {}
+
+  /**
+   * Get file information.
+   */
+  async stat(_filePath: string): Promise<Stats> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Check if file exists
+   */
+  async exists(filePath: string): Promise<boolean> {
+    try {
+      await this.stat(filePath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Create a readable stream to get the content of a file
+   * @deprecated Use {@link OcflStore.createReadable} instead
+   */
+  async createReadStream(_filePath: string, _options?: unknown): Promise<ReadStream> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Create a stream that writes data to `filePath`.
+   * @deprecated Use {@link OcflStore.createWritable} instead
+   */
+  async createWriteStream(_filePath: string, _options?: unknown): Promise<WriteStream> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Create a ReadableStream to get the content of a file
+   * @param filePath - Absolute path
+   */
+  async createReadable(_filePath: string, _options?: unknown): Promise<ReadableStream> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Create a WritableStream that writes data to `filePath`.
+   */
+  async createWritable(_filePath: string, _options?: unknown): Promise<WritableStream> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Provide a common interface to read a file inside an object.
+   * A concrete subclass MAY implement this method to provide read access to its underlying storage backend.
+   * The default implementation uses the {@linkcode OcflStore#createReadable} method.
+   */
+  async readFile(filePath: string, options: Encoding): Promise<string>;
+  async readFile(filePath: string, options?: unknown): Promise<Uint8Array>;
+  async readFile(filePath: string, options?: unknown): Promise<string | Uint8Array> {
+    const rs = await this.createReadable(filePath, options);
+    let buffer: string | Uint8Array =
+      typeof options === 'string' || (options as { encoding?: string } | undefined)?.encoding
+        ? ''
+        : new Uint8Array(new ArrayBuffer(0, { maxByteLength: 16 * 1024 }));
+    for await (const chunk of rs) {
+      if (typeof chunk === 'string') {
+        if (typeof buffer !== 'string') {
+          buffer = new TextDecoder().decode(buffer);
+        }
+        buffer += chunk;
+      } else {
+        if (typeof buffer === 'string') {
+          buffer = new TextEncoder().encode(buffer);
+        }
+        buffer = joinTypedArray(buffer, chunk) as Uint8Array;
+      }
+    }
+    return buffer;
+  }
+
+  /**
+   * Provide a common interface to write a file inside an object.
+   * A concrete subclass MAY implement this method to provide write access to its underlying storage backend.
+   * The default implementation uses the {@link OcflStore.createWritable} method.
+   * @param filePath - Absolute path to the file.
+   * @param data - The data to be written to the file.
+   */
+  async writeFile(filePath: string, data: WriteFileData, options?: object | string): Promise<void> {
+    await this.mkdir(path.dirname(filePath));
+    const target = await this.createWritable(filePath, options);
+    if (data instanceof ReadableStream) {
+      await data.pipeTo(target);
+    } else {
+      const w = target.getWriter();
+      await w.write(data);
+      await w.close();
+    }
+  }
+
+  /**
+   * Copy file from `source` to `target`.
+   * By default, target is overwritten if it already exists.
+   */
+  async copyFile(source: string, target: string): Promise<void> {
+    await this.writeFile(target, await this.createReadable(source));
+  }
+
+  /**
+   * Open a directory
+   */
+  async opendir(_filePath: string, _options?: unknown): Promise<Dir> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Read a directory
+   */
+  async readdir(_filePath: string, _options?: unknown): Promise<string[]> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * List all files contained in a directory including the metadata such as size and last modified date.
+   * The returned file.path is a relative path to the dirPath
+   */
+  async list(_dirPath: string, _options?: ListOptions): Promise<AsyncIterableIterator<ListedFile>> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Recursively create a directory if the storage backend supports it, otherwise do nothing
+   * @returns The first directory path created
+   */
+  async mkdir(_filePath: string, _options?: unknown): Promise<string | undefined> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Rename or move a file or directory
+   */
+  async move(_source: string, _target: string): Promise<void> {
+    throw new Error('Not Implemented');
+  }
+
+  /**
+   * Remove a file or directory recursively
+   */
+  async remove(_filePath: string): Promise<void> {
+    throw new Error('Not Implemented');
+  }
+}
+
+export { OcflStore };