@casfa/storage-fs 0.2.0 → 0.3.0

package/README.md

@@ -0,0 +1,91 @@
+# @casfa/storage-fs
+
+File system storage provider for CAS.
+
+## Installation
+
+```bash
+bun add @casfa/storage-fs
+```
+
+## Overview
+
+A file system-backed storage provider for CAS (Content-Addressable Storage). Stores content in a sharded directory structure for efficient file system performance.
+
+### Storage Structure
+
+```
+{basePath}/
+├── ab/
+│   └── cd/
+│       └── abcd1234...  (full hash)
+├── ef/
+│   └── 01/
+│       └── ef012345...
+└── ...
+```
+
+## Usage
+
+### Basic Usage
+
+```typescript
+import { createFsStorage } from '@casfa/storage-fs';
+
+const storage = createFsStorage({
+  basePath: '/var/cas/data',
+});
+
+// Store data
+await storage.put('node:abcd1234...', data);
+
+// Retrieve data
+const data = await storage.get('node:abcd1234...');
+```
+
+## Configuration
+
+```typescript
+interface FsStorageConfig {
+  // Required: Base directory for storage
+  basePath: string;
+  
+  // Optional: File permissions (default: 0o644)
+  fileMode?: number;
+  
+  // Optional: Directory permissions (default: 0o755)
+  dirMode?: number;
+}
+```
+
+## API Reference
+
+### Functions
+
+- `createFsStorage(config)` - Create file-system storage
+
+### StorageProvider Interface
+
+```typescript
+interface StorageProvider {
+  get(key: string): Promise<Uint8Array | null>;
+  put(key: string, data: Uint8Array): Promise<void>;
+}
+```
+
+## Performance Tips
+
+1. **Use caching** for read-heavy workloads
+2. **Use SSD storage** for better random access performance
+3. **Monitor disk space** as CAS data is append-only by nature
+4. Consider **separate partitions** for CAS data to prevent filling system disk
+
+## Related Packages
+
+- `@casfa/storage-core` - Core types and utilities
+- `@casfa/storage-memory` - In-memory storage (for testing)
+- `@casfa/storage-s3` - S3 storage (for cloud deployment)
+
+## License
+
+MIT

package/README.zh-CN.md

@@ -0,0 +1,91 @@
+# @casfa/storage-fs
+
+基于文件系统的 CAS 存储提供者。
+
+## 安装
+
+```bash
+bun add @casfa/storage-fs
+```
+
+## 概述
+
+基于文件系统的 CAS（内容寻址存储）存储提供者。采用分片目录结构存储内容，以获得更好的文件系统性能。
+
+### 存储结构
+
+```
+{basePath}/
+├── ab/
+│   └── cd/
+│       └── abcd1234...  (full hash)
+├── ef/
+│   └── 01/
+│       └── ef012345...
+└── ...
+```
+
+## 使用方法
+
+### 基本用法
+
+```typescript
+import { createFsStorage } from '@casfa/storage-fs';
+
+const storage = createFsStorage({
+  basePath: '/var/cas/data',
+});
+
+// 存储数据
+await storage.put('node:abcd1234...', data);
+
+// 检索数据
+const data = await storage.get('node:abcd1234...');
+```
+
+## 配置
+
+```typescript
+interface FsStorageConfig {
+  // 必需：存储基础目录
+  basePath: string;
+  
+  // 可选：文件权限（默认: 0o644）
+  fileMode?: number;
+  
+  // 可选：目录权限（默认: 0o755）
+  dirMode?: number;
+}
+```
+
+## API 参考
+
+### 函数
+
+- `createFsStorage(config)` - 创建文件系统存储
+
+### StorageProvider 接口
+
+```typescript
+interface StorageProvider {
+  get(key: string): Promise<Uint8Array | null>;
+  put(key: string, data: Uint8Array): Promise<void>;
+}
+```
+
+## 性能建议
+
+1. **使用缓存** 以应对读密集型工作负载
+2. **使用 SSD 存储** 以获得更好的随机访问性能
+3. **监控磁盘空间**，因为 CAS 数据本质上是只追加的
+4. 考虑为 CAS 数据使用**独立分区**，防止占满系统磁盘
+
+## 相关包
+
+- `@casfa/storage-core` - 核心类型与工具
+- `@casfa/storage-memory` - 内存存储（用于测试）
+- `@casfa/storage-s3` - S3 存储（用于云端部署）
+
+## 许可证
+
+MIT

package/dist/fs-storage.d.ts

@@ -0,0 +1,23 @@
+/**
+ * File System Storage Provider for CAS
+ *
+ * Implements StorageProvider with:
+ * - Local file system backend storage
+ * - Automatic directory creation
+ * - Internal existence check in put() to avoid redundant writes
+ */
+import type { StorageProvider } from "@casfa/storage-core";
+/**
+ * File System Storage configuration
+ */
+export type FsStorageConfig = {
+    /** Base directory for storage */
+    basePath: string;
+    /** Key prefix in storage (default: "cas/v1/") */
+    prefix?: string;
+};
+/**
+ * Create a file system-backed storage provider
+ */
+export declare const createFsStorage: (config: FsStorageConfig) => StorageProvider;
+//# sourceMappingURL=fs-storage.d.ts.map

package/dist/fs-storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fs-storage.d.ts","sourceRoot":"","sources":["../src/fs-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAa3D;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,eAAe,GAAI,QAAQ,eAAe,KAAG,eA6CzD,CAAC"}

package/dist/index.d.ts

@@ -1,40 +1,7 @@
-import { StorageProvider } from '@casfa/storage-core';
-
 /**
- * File System Storage Provider for CAS
+ * CAS File System Storage
  *
- * Implements StorageProvider with:
- * - LRU cache for key existence checks
- * - Local file system backend storage
- * - Automatic directory creation
+ * File system-backed storage provider for CAS.
  */
-
-/**
- * File System Storage configuration
- */
-type FsStorageConfig = {
-    /** Base directory for storage */
-    basePath: string;
-    /** LRU cache size for key existence (default: 10000) */
-    cacheSize?: number;
-    /** Key prefix in storage (default: "cas/sha256/") */
-    prefix?: string;
-};
-/**
- * Create a file system-backed storage provider
- */
-declare const createFsStorage: (config: FsStorageConfig) => StorageProvider;
-/**
- * Create file system storage with cache control methods (for testing)
- */
-declare const createFsStorageWithCache: (config: FsStorageConfig) => {
-    clearCache: () => void;
-    getCacheStats: () => {
-        size: number;
-    };
-    has: (key: string) => Promise<boolean>;
-    get: (key: string) => Promise<Uint8Array | null>;
-    put: (key: string, value: Uint8Array) => Promise<void>;
-};
-
-export { type FsStorageConfig, createFsStorage, createFsStorageWithCache };
+export { createFsStorage, type FsStorageConfig } from "./fs-storage.ts";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/index.js

@@ -1,39 +1,22 @@
 // src/fs-storage.ts
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { dirname, join } from "path";
-import {
-  createLRUCache,
-  DEFAULT_CACHE_SIZE,
-  toStoragePath
-} from "@casfa/storage-core";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { stat } from "node:fs/promises";
+import { dirname, join } from "node:path";
+var toStoragePath = (key, prefix) => {
+  const subdir = key.slice(0, 2);
+  return `${prefix}${subdir}/${key}`;
+};
 var createFsStorage = (config) => {
   const basePath = config.basePath;
-  const prefix = config.prefix ?? "cas/sha256/";
-  const existsCache = createLRUCache(config.cacheSize ?? DEFAULT_CACHE_SIZE);
-  if (!existsSync(basePath)) {
-    mkdirSync(basePath, { recursive: true });
-  }
+  const prefix = config.prefix ?? "cas/v1/";
   const toFilePath = (casKey) => {
     const storagePath = toStoragePath(casKey, prefix);
     return join(basePath, storagePath);
   };
-  const has = async (key) => {
-    const cached = existsCache.get(key);
-    if (cached !== void 0) {
-      return cached;
-    }
-    const filePath = toFilePath(key);
-    const exists = existsSync(filePath);
-    if (exists) {
-      existsCache.set(key, true);
-    }
-    return exists;
-  };
   const get = async (key) => {
     const filePath = toFilePath(key);
     try {
       const buffer = readFileSync(filePath);
-      existsCache.set(key, true);
       return new Uint8Array(buffer);
     } catch (error) {
       const err = error;
@@ -44,34 +27,19 @@ var createFsStorage = (config) => {
     }
   };
   const put = async (key, value) => {
-    if (existsCache.get(key)) {
-      return;
-    }
-    const exists = await has(key);
-    if (exists) {
-      return;
-    }
     const filePath = toFilePath(key);
+    try {
+      await stat(filePath);
+      return;
+    } catch {}
     const dir = dirname(filePath);
-    if (!existsSync(dir)) {
-      mkdirSync(dir, { recursive: true });
-    }
+    mkdirSync(dir, { recursive: true });
     writeFileSync(filePath, value);
-    existsCache.set(key, true);
-  };
-  return { has, get, put };
-};
-var createFsStorageWithCache = (config) => {
-  const existsCache = createLRUCache(config.cacheSize ?? DEFAULT_CACHE_SIZE);
-  const storage = createFsStorage(config);
-  return {
-    ...storage,
-    clearCache: () => existsCache.clear(),
-    getCacheStats: () => ({ size: existsCache.size() })
   };
+  return { get, put };
 };
 export {
-  createFsStorage,
-  createFsStorageWithCache
+  createFsStorage
 };
-//# sourceMappingURL=index.js.map
+
+//# debugId=21AA1842112D604764756E2164756E21

package/dist/index.js.map

@@ -1 +1,10 @@
-{"version":3,"sources":["../src/fs-storage.ts"],"sourcesContent":["/**\n * File System Storage Provider for CAS\n *\n * Implements StorageProvider with:\n * - LRU cache for key existence checks\n * - Local file system backend storage\n * - Automatic directory creation\n */\n\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport {\n  createLRUCache,\n  DEFAULT_CACHE_SIZE,\n  type StorageProvider,\n  toStoragePath,\n} from \"@casfa/storage-core\";\n\n/**\n * File System Storage configuration\n */\nexport type FsStorageConfig = {\n  /** Base directory for storage */\n  basePath: string;\n  /** LRU cache size for key existence (default: 10000) */\n  cacheSize?: number;\n  /** Key prefix in storage (default: \"cas/sha256/\") */\n  prefix?: string;\n};\n\n/**\n * Create a file system-backed storage provider\n */\nexport const createFsStorage = (config: FsStorageConfig): StorageProvider => {\n  const basePath = config.basePath;\n  const prefix = config.prefix ?? \"cas/sha256/\";\n  const existsCache = createLRUCache<string, boolean>(config.cacheSize ?? DEFAULT_CACHE_SIZE);\n\n  // Ensure base directory exists\n  if (!existsSync(basePath)) {\n    mkdirSync(basePath, { recursive: true });\n  }\n\n  const toFilePath = (casKey: string): string => {\n    const storagePath = toStoragePath(casKey, prefix);\n    return join(basePath, storagePath);\n  };\n\n  const has = async (key: string): Promise<boolean> => {\n    // Check cache first\n    const cached = existsCache.get(key);\n    if (cached !== undefined) {\n      return cached;\n    }\n\n    // Check file system\n    const filePath = toFilePath(key);\n    const exists = existsSync(filePath);\n\n    if (exists) {\n      existsCache.set(key, true);\n    }\n    // Don't cache non-existence (it might be written later)\n\n    return exists;\n  };\n\n  const get = async (key: string): Promise<Uint8Array | null> => {\n    const filePath = toFilePath(key);\n\n    try {\n      const buffer = readFileSync(filePath);\n      // Mark as existing in cache\n      existsCache.set(key, true);\n      return new Uint8Array(buffer);\n    } catch (error: unknown) {\n      const err = error as { code?: string };\n      if (err.code === \"ENOENT\") {\n        return null;\n      }\n      throw error;\n    }\n  };\n\n  const put = async (key: string, value: Uint8Array): Promise<void> => {\n    // Check cache first (avoid redundant writes)\n    if (existsCache.get(key)) {\n      return;\n    }\n\n    // Check if already exists\n    const exists = await has(key);\n    if (exists) {\n      return;\n    }\n\n    const filePath = toFilePath(key);\n    const dir = dirname(filePath);\n\n    // Ensure directory exists\n    if (!existsSync(dir)) {\n      mkdirSync(dir, { recursive: true });\n    }\n\n    // Write file\n    writeFileSync(filePath, value);\n\n    // Mark as existing\n    existsCache.set(key, true);\n  };\n\n  return { has, get, put };\n};\n\n/**\n * Create file system storage with cache control methods (for testing)\n */\nexport const createFsStorageWithCache = (config: FsStorageConfig) => {\n  const existsCache = createLRUCache<string, boolean>(config.cacheSize ?? DEFAULT_CACHE_SIZE);\n\n  const storage = createFsStorage(config);\n\n  return {\n    ...storage,\n    clearCache: () => existsCache.clear(),\n    getCacheStats: () => ({ size: existsCache.size() }),\n  };\n};\n"],"mappings":";AASA,SAAS,YAAY,WAAW,cAAc,qBAAqB;AACnE,SAAS,SAAS,YAAY;AAC9B;AAAA,EACE;AAAA,EACA;AAAA,EAEA;AAAA,OACK;AAiBA,IAAM,kBAAkB,CAAC,WAA6C;AAC3E,QAAM,WAAW,OAAO;AACxB,QAAM,SAAS,OAAO,UAAU;AAChC,QAAM,cAAc,eAAgC,OAAO,aAAa,kBAAkB;AAG1F,MAAI,CAAC,WAAW,QAAQ,GAAG;AACzB,cAAU,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,EACzC;AAEA,QAAM,aAAa,CAAC,WAA2B;AAC7C,UAAM,cAAc,cAAc,QAAQ,MAAM;AAChD,WAAO,KAAK,UAAU,WAAW;AAAA,EACnC;AAEA,QAAM,MAAM,OAAO,QAAkC;AAEnD,UAAM,SAAS,YAAY,IAAI,GAAG;AAClC,QAAI,WAAW,QAAW;AACxB,aAAO;AAAA,IACT;AAGA,UAAM,WAAW,WAAW,GAAG;AAC/B,UAAM,SAAS,WAAW,QAAQ;AAElC,QAAI,QAAQ;AACV,kBAAY,IAAI,KAAK,IAAI;AAAA,IAC3B;AAGA,WAAO;AAAA,EACT;AAEA,QAAM,MAAM,OAAO,QAA4C;AAC7D,UAAM,WAAW,WAAW,GAAG;AAE/B,QAAI;AACF,YAAM,SAAS,aAAa,QAAQ;AAEpC,kBAAY,IAAI,KAAK,IAAI;AACzB,aAAO,IAAI,WAAW,MAAM;AAAA,IAC9B,SAAS,OAAgB;AACvB,YAAM,MAAM;AACZ,UAAI,IAAI,SAAS,UAAU;AACzB,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,QAAM,MAAM,OAAO,KAAa,UAAqC;AAEnE,QAAI,YAAY,IAAI,GAAG,GAAG;AACxB;AAAA,IACF;AAGA,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV;AAAA,IACF;AAEA,UAAM,WAAW,WAAW,GAAG;AAC/B,UAAM,MAAM,QAAQ,QAAQ;AAG5B,QAAI,CAAC,WAAW,GAAG,GAAG;AACpB,gBAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,IACpC;AAGA,kBAAc,UAAU,KAAK;AAG7B,gBAAY,IAAI,KAAK,IAAI;AAAA,EAC3B;AAEA,SAAO,EAAE,KAAK,KAAK,IAAI;AACzB;AAKO,IAAM,2BAA2B,CAAC,WAA4B;AACnE,QAAM,cAAc,eAAgC,OAAO,aAAa,kBAAkB;AAE1F,QAAM,UAAU,gBAAgB,MAAM;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,YAAY,MAAM,YAAY,MAAM;AAAA,IACpC,eAAe,OAAO,EAAE,MAAM,YAAY,KAAK,EAAE;AAAA,EACnD;AACF;","names":[]}
+{
+  "version": 3,
+  "sources": ["../src/fs-storage.ts"],
+  "sourcesContent": [
+    "/**\n * File System Storage Provider for CAS\n *\n * Implements StorageProvider with:\n * - Local file system backend storage\n * - Automatic directory creation\n * - Internal existence check in put() to avoid redundant writes\n */\n\nimport { mkdirSync, readFileSync, writeFileSync } from \"node:fs\";\nimport { stat } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport type { StorageProvider } from \"@casfa/storage-core\";\n\n/**\n * Create storage path from a CB32 storage key.\n * Uses first 2 chars as subdirectory for better distribution.\n *\n * Example: 240B5PHBGEC2A705WTKKMVRS30 -> cas/v1/24/240B5PHBGEC2A705WTKKMVRS30\n */\nconst toStoragePath = (key: string, prefix: string): string => {\n  const subdir = key.slice(0, 2);\n  return `${prefix}${subdir}/${key}`;\n};\n\n/**\n * File System Storage configuration\n */\nexport type FsStorageConfig = {\n  /** Base directory for storage */\n  basePath: string;\n  /** Key prefix in storage (default: \"cas/v1/\") */\n  prefix?: string;\n};\n\n/**\n * Create a file system-backed storage provider\n */\nexport const createFsStorage = (config: FsStorageConfig): StorageProvider => {\n  const basePath = config.basePath;\n  const prefix = config.prefix ?? \"cas/v1/\";\n\n  const toFilePath = (casKey: string): string => {\n    const storagePath = toStoragePath(casKey, prefix);\n    return join(basePath, storagePath);\n  };\n\n  const get = async (key: string): Promise<Uint8Array | null> => {\n    const filePath = toFilePath(key);\n\n    try {\n      const buffer = readFileSync(filePath);\n      return new Uint8Array(buffer);\n    } catch (error: unknown) {\n      const err = error as { code?: string };\n      if (err.code === \"ENOENT\") {\n        return null;\n      }\n      throw error;\n    }\n  };\n\n  const put = async (key: string, value: Uint8Array): Promise<void> => {\n    const filePath = toFilePath(key);\n\n    // Internal optimization: stat is cheaper than write for existing files\n    try {\n      await stat(filePath);\n      return; // already exists\n    } catch {\n      // doesn't exist, proceed to write\n    }\n\n    const dir = dirname(filePath);\n\n    // Ensure directory exists\n    mkdirSync(dir, { recursive: true });\n\n    // Write file\n    writeFileSync(filePath, value);\n  };\n\n  return { get, put };\n};\n"
+  ],
+  "mappings": ";AASA;AACA;AACA;AASA,IAAM,gBAAgB,CAAC,KAAa,WAA2B;AAAA,EAC7D,MAAM,SAAS,IAAI,MAAM,GAAG,CAAC;AAAA,EAC7B,OAAO,GAAG,SAAS,UAAU;AAAA;AAgBxB,IAAM,kBAAkB,CAAC,WAA6C;AAAA,EAC3E,MAAM,WAAW,OAAO;AAAA,EACxB,MAAM,SAAS,OAAO,UAAU;AAAA,EAEhC,MAAM,aAAa,CAAC,WAA2B;AAAA,IAC7C,MAAM,cAAc,cAAc,QAAQ,MAAM;AAAA,IAChD,OAAO,KAAK,UAAU,WAAW;AAAA;AAAA,EAGnC,MAAM,MAAM,OAAO,QAA4C;AAAA,IAC7D,MAAM,WAAW,WAAW,GAAG;AAAA,IAE/B,IAAI;AAAA,MACF,MAAM,SAAS,aAAa,QAAQ;AAAA,MACpC,OAAO,IAAI,WAAW,MAAM;AAAA,MAC5B,OAAO,OAAgB;AAAA,MACvB,MAAM,MAAM;AAAA,MACZ,IAAI,IAAI,SAAS,UAAU;AAAA,QACzB,OAAO;AAAA,MACT;AAAA,MACA,MAAM;AAAA;AAAA;AAAA,EAIV,MAAM,MAAM,OAAO,KAAa,UAAqC;AAAA,IACnE,MAAM,WAAW,WAAW,GAAG;AAAA,IAG/B,IAAI;AAAA,MACF,MAAM,KAAK,QAAQ;AAAA,MACnB;AAAA,MACA,MAAM;AAAA,IAIR,MAAM,MAAM,QAAQ,QAAQ;AAAA,IAG5B,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,IAGlC,cAAc,UAAU,KAAK;AAAA;AAAA,EAG/B,OAAO,EAAE,KAAK,IAAI;AAAA;",
+  "debugId": "21AA1842112D604764756E2164756E21",
+  "names": []
+}

package/package.json

@@ -1,19 +1,19 @@
 {
   "name": "@casfa/storage-fs",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "File system storage provider for CAS",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "exports": {
     ".": {
+      "bun": "./src/index.ts",
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     }
   },
   "scripts": {
-    "build": "tsup",
-    "dev": "tsup --watch",
+    "build": "bun ../../scripts/build-pkg.ts",
     "typecheck": "tsc --noEmit",
     "lint": "biome check .",
     "lint:fix": "biome check --write .",