gh-as-db 1.2.0 → 1.3.0

package/README.md

@@ -11,6 +11,7 @@ Use a private GitHub repository as a database for your application. `gh-as-db` p
 - 🔐 **Secure**: Designed for private repositories using Personal Access Tokens (PAT).
 - 🚀 **Performance**: Built-in in-memory caching and auto-indexing for fast local queries.
 - 🛡️ **Concurrency**: Optimistic locking using Git SHAs to prevent data loss.
+- 🔄 **Retry & Rate Limits**: Automatic retries with exponential backoff for transient errors and GitHub rate limits.
 - 🔗 **Transactions**: Group multiple operations into a single atomic Git commit.
 - 📁 **Sharding**: One-file-per-document storage strategy for massive collections.
 - 🧩 **Middleware**: Extensible hooks for data validation or transformation.
@@ -22,7 +23,7 @@ Use a private GitHub repository as a database for your application. `gh-as-db` p
 ## Installation
 
 ```bash
-npm install gh-as-db
+pnpm add gh-as-db
 ```
 
 ## Quick Start
@@ -85,6 +86,7 @@ const db = new GitHubDB({
   owner: string;       // Repository owner
   repo: string;        // Repository name
   cacheTTL?: number;   // Optional: Cache TTL in ms. Default is 0 (strict consistency).
+  retry?: RetryConfig | false; // Optional: Retry config, or false to disable. See below.
 });
 ```
 
@@ -193,6 +195,49 @@ const users = db.collection<User>('users', {
 - 📈 **Scalability**: Avoids GitHub's file size limits and reduces merge conflicts.
 - 🧹 **Cleanliness**: Better organization for repositories with thousands of documents.
 
+### Retry & Rate Limit Handling
+
+All GitHub API calls are automatically retried on transient errors (429 rate limits, 500/502/503 server errors) with exponential backoff. If GitHub returns a `Retry-After` header, it is respected.
+
+```typescript
+const db = new GitHubDB({
+  accessToken: process.env.GITHUB_TOKEN,
+  owner: 'your-username',
+  repo: 'my-data-repo',
+  retry: {
+    maxRetries: 3,   // Default: 3
+    baseDelay: 1000, // Default: 1000ms
+    maxDelay: 10000, // Default: 10000ms
+  }
+});
+```
+
+To disable retries entirely:
+
+```typescript
+const db = new GitHubDB({
+  // ...
+  retry: false,
+});
+```
+
+Non-transient errors (401, 403, 404) are thrown immediately without retrying. Concurrency conflicts (409) still throw `ConcurrencyError` immediately. If rate limit retries are exhausted, a `RateLimitError` is thrown.
+
+```typescript
+import { RateLimitError, ConcurrencyError } from 'gh-as-db';
+
+try {
+  await users.create({ id: '1', name: 'Alice' });
+} catch (error) {
+  if (error instanceof RateLimitError) {
+    console.log('Rate limited, retry after:', error.retryAfter);
+  }
+  if (error instanceof ConcurrencyError) {
+    console.log('Conflict, re-read and retry');
+  }
+}
+```
+
 ## CLI Usage
 
 `gh-as-db` comes with a CLI tool to help you manage your repository.
@@ -221,6 +266,7 @@ For small projects, side-projects, or internal tools, setting up a database serv
 - **Write-Through**: Updates the local cache immediately after a write, preventing 404s during redirects.
 - **Indexing**: Automatic in-memory indexing on all fields makes querying fast even as data grows.
 - **Optimistic Concurrency**: Uses Git SHAs to ensure that you don't overwrite changes made by another client.
+- **Automatic Retries**: Transient failures and rate limits are handled transparently with exponential backoff.
 
 ## License
 

package/dist/core/types.d.ts

@@ -1,9 +1,15 @@
+export interface RetryConfig {
+    maxRetries?: number;
+    baseDelay?: number;
+    maxDelay?: number;
+}
 export interface GitHubDBConfig {
     accessToken: string;
     owner: string;
     repo: string;
     branch?: string;
     cacheTTL?: number;
+    retry?: RetryConfig | false;
 }
 export type Schema = Record<string, any>;
 export type MiddlewareOperation = "create" | "update" | "read" | "delete";
@@ -39,6 +45,10 @@ export declare class ConcurrencyError extends Error {
     readonly path: string;
     constructor(path: string);
 }
+export declare class RateLimitError extends Error {
+    readonly retryAfter?: number | undefined;
+    constructor(retryAfter?: number | undefined);
+}
 export interface StorageResponse<T> {
     data: T;
     sha: string;

package/dist/core/types.js

@@ -6,3 +6,11 @@ export class ConcurrencyError extends Error {
         this.name = "ConcurrencyError";
     }
 }
+export class RateLimitError extends Error {
+    retryAfter;
+    constructor(retryAfter) {
+        super(`GitHub API rate limit exceeded${retryAfter ? `. Retry after ${retryAfter}s` : ""}`);
+        this.retryAfter = retryAfter;
+        this.name = "RateLimitError";
+    }
+}

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { GitHubDB } from "./ui/github-db.js";
 export { Collection } from "./ui/collection.js";
 export { Transaction } from "./ui/transaction.js";
 export * from "./core/types.js";
-export declare const version = "1.2.0";
+export declare const version = "1.3.0";

package/dist/index.js

@@ -2,4 +2,4 @@ export { GitHubDB } from "./ui/github-db.js";
 export { Collection } from "./ui/collection.js";
 export { Transaction } from "./ui/transaction.js";
 export * from "./core/types.js";
-export const version = "1.2.0";
+export const version = "1.3.0";

package/dist/infrastructure/github-storage.d.ts

@@ -7,6 +7,7 @@ export declare class GitHubStorageProvider implements IStorageProvider {
     private staleCache;
     private readonly DEFAULT_TTL;
     constructor(config: GitHubDBConfig, cache?: ICacheProvider);
+    private retryWithBackoff;
     testConnection(): Promise<boolean>;
     exists(path: string): Promise<boolean>;
     readJson<T>(path: string): Promise<StorageResponse<T>>;

package/dist/infrastructure/github-storage.js

@@ -1,5 +1,5 @@
 import { Octokit } from "@octokit/rest";
-import { ConcurrencyError, } from "../core/types.js";
+import { ConcurrencyError, RateLimitError, } from "../core/types.js";
 import { MemoryCacheProvider } from "./cache-provider.js";
 export class GitHubStorageProvider {
     config;
@@ -14,12 +14,58 @@ export class GitHubStorageProvider {
         });
         this.cache = cache || new MemoryCacheProvider();
     }
+    async retryWithBackoff(fn) {
+        if (this.config.retry === false)
+            return fn();
+        const { maxRetries = 3, baseDelay = 1000, maxDelay = 10000, } = this.config.retry ?? {};
+        let lastError;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                return await fn();
+            }
+            catch (error) {
+                lastError = error;
+                const status = error?.status;
+                // Non-retryable: 409 (concurrency), 4xx client errors (except 429)
+                if (status === 409)
+                    throw error;
+                if (status && status < 500 && status !== 429)
+                    throw error;
+                // Errors without a status code (non-HTTP errors) — rethrow
+                if (!status)
+                    throw error;
+                // Last attempt — don't wait, just break
+                if (attempt === maxRetries)
+                    break;
+                // Calculate delay
+                let delay;
+                if (status === 429 && error.response?.headers?.["retry-after"]) {
+                    delay =
+                        parseInt(error.response.headers["retry-after"], 10) * 1000;
+                }
+                else {
+                    delay = baseDelay * Math.pow(2, attempt);
+                    // Add jitter to avoid thundering herd on simultaneous retries
+                    delay = delay * (0.5 + Math.random() * 0.5);
+                }
+                // Cap all delays (including Retry-After) against maxDelay
+                delay = Math.min(delay, maxDelay);
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
+        }
+        // Exhausted retries on 429 → RateLimitError
+        if (lastError.status === 429) {
+            const retryAfter = lastError.response?.headers?.["retry-after"];
+            throw new RateLimitError(retryAfter ? parseInt(retryAfter, 10) : undefined);
+        }
+        throw lastError;
+    }
     async testConnection() {
         try {
-            await this.octokit.repos.get({
+            await this.retryWithBackoff(() => this.octokit.repos.get({
                 owner: this.config.owner,
                 repo: this.config.repo,
-            });
+            }));
             return true;
         }
         catch (error) {
@@ -31,11 +77,11 @@ export class GitHubStorageProvider {
             return true;
         }
         try {
-            await this.octokit.repos.getContent({
+            await this.retryWithBackoff(() => this.octokit.repos.getContent({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 path,
-            });
+            }));
             return true;
         }
         catch (error) {
@@ -54,12 +100,12 @@ export class GitHubStorageProvider {
         // 2. Check stale cache for conditional request
         const stale = this.staleCache.get(path);
         try {
-            const response = await this.octokit.repos.getContent({
+            const response = await this.retryWithBackoff(() => this.octokit.repos.getContent({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 path,
                 headers: stale ? { "if-none-match": `"${stale.sha}"` } : {},
-            });
+            }));
             if (Array.isArray(response.data)) {
                 throw new Error("Path is a directory, not a file");
             }
@@ -102,11 +148,11 @@ export class GitHubStorageProvider {
             }
             else {
                 try {
-                    const existing = await this.octokit.repos.getContent({
+                    const existing = await this.retryWithBackoff(() => this.octokit.repos.getContent({
                         owner: this.config.owner,
                         repo: this.config.repo,
                         path,
-                    });
+                    }));
                     if (!Array.isArray(existing.data) && "sha" in existing.data) {
                         internalSha = existing.data.sha;
                     }
@@ -118,8 +164,11 @@ export class GitHubStorageProvider {
                 }
             }
         }
+        // Note: internalSha is resolved before the retry wrapper. If a write fails
+        // with a 5xx after GitHub has already committed the file (e.g. network timeout),
+        // the retry will reuse the stale sha and may fail with a 409.
         try {
-            const response = await this.octokit.repos.createOrUpdateFileContents({
+            const response = await this.retryWithBackoff(() => this.octokit.repos.createOrUpdateFileContents({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 path,
@@ -128,7 +177,7 @@ export class GitHubStorageProvider {
                     .map((b) => String.fromCharCode(b))
                     .join("")),
                 sha: internalSha,
-            });
+            }));
             const newSha = response.data.content?.sha || "";
             const result = { data: content, sha: newSha };
             const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
@@ -150,18 +199,18 @@ export class GitHubStorageProvider {
         const branch = this.config.branch || "main";
         try {
             // 1. Get the current head SHA
-            const { data: ref } = await this.octokit.git.getRef({
+            const { data: ref } = await this.retryWithBackoff(() => this.octokit.git.getRef({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 ref: `heads/${branch}`,
-            });
+            }));
             const latestCommitSha = ref.object.sha;
             // 2. Get the tree SHA of the latest commit
-            const { data: latestCommit } = await this.octokit.git.getCommit({
+            const { data: latestCommit } = await this.retryWithBackoff(() => this.octokit.git.getCommit({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 commit_sha: latestCommitSha,
-            });
+            }));
             const baseTreeSha = latestCommit.tree.sha;
             // 3. Create a new tree with multiple files
             const treeEntries = changes.map((change) => {
@@ -180,27 +229,27 @@ export class GitHubStorageProvider {
                     content: JSON.stringify(change.content, null, 2),
                 };
             });
-            const { data: newTree } = await this.octokit.git.createTree({
+            const { data: newTree } = await this.retryWithBackoff(() => this.octokit.git.createTree({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 base_tree: baseTreeSha,
                 tree: treeEntries,
-            });
+            }));
             // 4. Create a new commit
-            const { data: newCommit } = await this.octokit.git.createCommit({
+            const { data: newCommit } = await this.retryWithBackoff(() => this.octokit.git.createCommit({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 message,
                 tree: newTree.sha,
                 parents: [latestCommitSha],
-            });
+            }));
             // 5. Update the reference
-            await this.octokit.git.updateRef({
+            await this.retryWithBackoff(() => this.octokit.git.updateRef({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 ref: `heads/${branch}`,
                 sha: newCommit.sha,
-            });
+            }));
             // 6. Update cache for all involved files
             const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
             for (const entry of newTree.tree) {
@@ -223,23 +272,23 @@ export class GitHubStorageProvider {
         }
     }
     async deleteFile(path, message, sha) {
-        await this.octokit.repos.deleteFile({
+        await this.retryWithBackoff(() => this.octokit.repos.deleteFile({
             owner: this.config.owner,
             repo: this.config.repo,
             path,
             message,
             sha,
-        });
+        }));
         this.cache.delete(path);
         this.staleCache.delete(path);
     }
     async listDirectory(path) {
         try {
-            const response = await this.octokit.repos.getContent({
+            const response = await this.retryWithBackoff(() => this.octokit.repos.getContent({
                 owner: this.config.owner,
                 repo: this.config.repo,
                 path,
-            });
+            }));
             if (!Array.isArray(response.data)) {
                 throw new Error("Path is not a directory");
             }

package/dist/ui/github-db.d.ts

@@ -1,4 +1,4 @@
-import { GitHubDBConfig, IStorageProvider, Middleware, Schema, Validator } from "../core/types.js";
+import { GitHubDBConfig, IStorageProvider, Middleware, Schema, StorageStrategy, Validator } from "../core/types.js";
 import { Collection } from "./collection.js";
 import { Transaction } from "./transaction.js";
 export declare class GitHubDB {
@@ -9,6 +9,7 @@ export declare class GitHubDB {
     collection<T extends Schema>(name: string, options?: {
         middleware?: Middleware<T>[];
         validator?: Validator<T>;
+        strategy?: StorageStrategy;
     }): Collection<T>;
     transaction(fn: (tx: Transaction) => Promise<void>, message?: string): Promise<string>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gh-as-db",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Use a private GitHub repository as a database for your application.",
   "type": "module",
   "main": "./dist/index.js",
@@ -20,12 +20,6 @@
     "README.md",
     "LICENSE"
   ],
-  "scripts": {
-    "test": "vitest",
-    "test:run": "vitest run",
-    "build": "tsc",
-    "prepublishOnly": "npm run build"
-  },
   "keywords": [
     "github",
     "database",
@@ -60,5 +54,10 @@
     "chalk": "^5.4.1",
     "commander": "^13.1.0",
     "enquirer": "^2.4.1"
+  },
+  "scripts": {
+    "test": "vitest",
+    "test:run": "vitest run",
+    "build": "tsc"
   }
-}
+}