gh-as-db 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md ADDED
@@ -0,0 +1,51 @@
1
+ # Changelog
2
+
3
+ ## [2.0.0] — 2026-07-17
4
+
5
+ ### Breaking Changes
6
+
7
+ - **`transaction()` return type**: `Promise<string>` → `Promise<string | null>`. Empty transactions now return `null` instead of `""`. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
8
+ - **Collection name validation**: Names must match `[A-Za-z0-9._-]` (max 255 chars). Names with spaces, slashes, or special characters now throw. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
9
+ - **Item ID validation**: IDs are validated on `create()`, `findById()`, `update()`, and `delete()`. IDs must match `[A-Za-z0-9._-]` (max 255 chars). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
10
+ - **Unknown filter operators now throw** instead of silently skipping. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
11
+ - **`"in"` filter operator requires an array value** — non-array values now throw. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
12
+ - **Invalid pagination throws**: negative/infinite `offset` or `limit` now throws. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
13
+ - **Sharded items: id is now immutable** — `update()` with a different id on sharded collections throws. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
14
+ - **`create()` no longer silently swallows transient read errors** — prevents data loss from overwriting entire collections. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
15
+
16
+ ### Added
17
+
18
+ - **Path traversal protection**: Collection names, item IDs, and commit paths validated against `../`, null bytes, and path separators. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
19
+ - **Prototype pollution hardening**: `__proto__`, `constructor`, and `prototype` keys rejected in filter fields and update payloads. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
20
+ - **Network error retry**: ENOTFOUND, ECONNRESET, ETIMEDOUT, ECONNREFUSED, EPIPE, and other network errors are retried with exponential backoff. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
21
+ - **Secondary rate limit detection**: 403 responses with `Retry-After` header or secondary rate limit message are retried. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
22
+ - **`Retry-After` HTTP-date parsing**: Falls back to exponential backoff on malformed values. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
23
+ - **Real ETag capture** for conditional `GET` requests (`If-None-Match`). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
24
+ - **`CorruptedContentError`**: Explicit error for malformed JSON or binary content. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
25
+ - **`toJSON()` on `GitHubDB` and `GitHubStorageProvider`** — accessToken redacted for safe logging. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
26
+ - **CLI: sharded collection detection** — `list` and `inspect` commands detect directory-based (sharded) collections. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
27
+ - **CLI: detailed connection test** — categorized failure reasons (auth, not-found, rate-limit, network). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
28
+
29
+ ### Fixed
30
+
31
+ - **`afterRead` middleware now applies on all read paths** — including cached, indexed, and fast-path reads. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
32
+ - **Items stored as raw data** — `afterRead` transformations are no longer persisted to storage. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
33
+ - **`update()` works on immutable copies** — no longer mutates in-memory items in-place. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
34
+ - **Sharded upsert uses `indexer.update()`** (not bare `add()`) — fixes stale index entries. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
35
+ - **`deleteFile` wraps 409 → `ConcurrencyError`**; **`commit()` maps 422 → `ConcurrencyError`**. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
36
+ - **`exists()` only trusts fresh cache**; stale cache bounded to 1000 entries (LRU). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
37
+ - **Transaction isolation**: `listDirectory` merges pending creates/deletes; `exists()` handles pending deletes. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
38
+ - **CLI error messages genericized** — no raw API errors leaked to terminal. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
39
+
40
+ ### Security
41
+
42
+ - Path traversal hardening (CWE-22)
43
+ - Prototype pollution hardening (CWE-1321)
44
+ - accessToken redaction in `toJSON()` / logging
45
+ - CLI error message sanitization
46
+
47
+ ---
48
+
49
+ ## [1.3.0] — 2025-12-03
50
+
51
+ - Initial public release
package/README.md CHANGED
@@ -11,6 +11,7 @@ Use a private GitHub repository as a database for your application. `gh-as-db` p
11
11
  - 🔐 **Secure**: Designed for private repositories using Personal Access Tokens (PAT).
12
12
  - 🚀 **Performance**: Built-in in-memory caching and auto-indexing for fast local queries.
13
13
  - 🛡️ **Concurrency**: Optimistic locking using Git SHAs to prevent data loss.
14
+ - 🔄 **Retry & Rate Limits**: Automatic retries with exponential backoff for transient errors and GitHub rate limits.
14
15
  - 🔗 **Transactions**: Group multiple operations into a single atomic Git commit.
15
16
  - 📁 **Sharding**: One-file-per-document storage strategy for massive collections.
16
17
  - 🧩 **Middleware**: Extensible hooks for data validation or transformation.
@@ -22,7 +23,7 @@ Use a private GitHub repository as a database for your application. `gh-as-db` p
22
23
  ## Installation
23
24
 
24
25
  ```bash
25
- npm install gh-as-db
26
+ pnpm add gh-as-db
26
27
  ```
27
28
 
28
29
  ## Quick Start
@@ -43,6 +44,7 @@ const db = new GitHubDB({
43
44
  });
44
45
 
45
46
  // Access the 'users' collection
47
+ // Note: avoid logging or serializing the db instance — the access token is stored on db.config
46
48
  const users = db.collection<User>('users');
47
49
 
48
50
  // Create a new user
@@ -84,7 +86,8 @@ const db = new GitHubDB({
84
86
  accessToken: string; // GitHub PAT with 'repo' scope
85
87
  owner: string; // Repository owner
86
88
  repo: string; // Repository name
87
- cacheTTL?: number; // Optional: Cache TTL in ms. Default is 0 (strict consistency).
89
+ cacheTTL?: number; // Optional: Cache TTL in ms. Default is 0 (strict consistency — always validates with GitHub). Set to a positive value (e.g. 5000) to enable in-memory caching for performance. Note: cacheTTL > 0 may return stale data across instances until the TTL expires.
90
+ retry?: RetryConfig | false; // Optional: Retry config, or false to disable. See below.
88
91
  });
89
92
  ```
90
93
 
@@ -193,6 +196,51 @@ const users = db.collection<User>('users', {
193
196
  - 📈 **Scalability**: Avoids GitHub's file size limits and reduces merge conflicts.
194
197
  - 🧹 **Cleanliness**: Better organization for repositories with thousands of documents.
195
198
 
199
+ **Important**: In sharded mode, item `id` values cannot be changed via `update()`. To change an id, delete the item and re-create it with the new id.
200
+
201
+ ### Retry & Rate Limit Handling
202
+
203
+ All GitHub API calls are automatically retried on transient errors (429 rate limits, 500/502/503 server errors) with exponential backoff. If GitHub returns a `Retry-After` header, it is respected.
204
+
205
+ ```typescript
206
+ const db = new GitHubDB({
207
+ accessToken: process.env.GITHUB_TOKEN,
208
+ owner: 'your-username',
209
+ repo: 'my-data-repo',
210
+ retry: {
211
+ maxRetries: 3, // Default: 3
212
+ baseDelay: 1000, // Default: 1000ms
213
+ maxDelay: 10000, // Default: 10000ms
214
+ }
215
+ });
216
+ ```
217
+
218
+ To disable retries entirely:
219
+
220
+ ```typescript
221
+ const db = new GitHubDB({
222
+ // ...
223
+ retry: false,
224
+ });
225
+ ```
226
+
227
+ Non-transient errors (401, 404, and permission-related 403) are thrown immediately. GitHub secondary rate limits (abuse-detection 403 responses) are retried automatically with backoff, just like 429 rate limits. Concurrency conflicts (409) still throw `ConcurrencyError` immediately. If rate limit retries are exhausted, a `RateLimitError` is thrown.
228
+
229
+ ```typescript
230
+ import { RateLimitError, ConcurrencyError } from 'gh-as-db';
231
+
232
+ try {
233
+ await users.create({ id: '1', name: 'Alice' });
234
+ } catch (error) {
235
+ if (error instanceof RateLimitError) {
236
+ console.log('Rate limited, retry after:', error.retryAfter);
237
+ }
238
+ if (error instanceof ConcurrencyError) {
239
+ console.log('Conflict, re-read and retry');
240
+ }
241
+ }
242
+ ```
243
+
196
244
  ## CLI Usage
197
245
 
198
246
  `gh-as-db` comes with a CLI tool to help you manage your repository.
@@ -217,10 +265,17 @@ For small projects, side-projects, or internal tools, setting up a database serv
217
265
 
218
266
  ## Performance
219
267
 
220
- - **Consistent Caching**: Uses **Conditional GET** (`If-None-Match`) to ensure data is always up-to-date even across multiple instances (e.g., serverless), while minimizing API costs.
268
+ - **Consistent Caching**: With the default `cacheTTL: 0`, every read validates against GitHub using **Conditional GET** (`If-None-Match`) to ensure data is always up-to-date even across multiple instances (e.g., serverless), while minimizing API costs. Set `cacheTTL > 0` for additional in-memory caching with a possible staleness window.
221
269
  - **Write-Through**: Updates the local cache immediately after a write, preventing 404s during redirects.
222
270
  - **Indexing**: Automatic in-memory indexing on all fields makes querying fast even as data grows.
223
271
  - **Optimistic Concurrency**: Uses Git SHAs to ensure that you don't overwrite changes made by another client.
272
+ - **Automatic Retries**: Transient failures and rate limits are handled transparently with exponential backoff.
273
+
274
+ ## Security
275
+
276
+ - **Token handling**: The GitHub PAT is stored on `db.config.accessToken`. Avoid logging, serializing, or exposing the `db` object. The library's `toJSON()` method redacts the token if you need to serialize the config for debugging.
277
+ - **Path safety**: Collection names and item IDs are validated against path traversal attacks. Only alphanumeric characters, dots, hyphens, and underscores are permitted.
278
+ - **Input validation**: Query operators are validated and unknown operators throw errors. Pagination parameters are range-checked.
224
279
 
225
280
  ## License
226
281
 
@@ -1,9 +1,15 @@
1
+ export interface RetryConfig {
2
+ maxRetries?: number;
3
+ baseDelay?: number;
4
+ maxDelay?: number;
5
+ }
1
6
  export interface GitHubDBConfig {
2
7
  accessToken: string;
3
8
  owner: string;
4
9
  repo: string;
5
10
  branch?: string;
6
11
  cacheTTL?: number;
12
+ retry?: RetryConfig | false;
7
13
  }
8
14
  export type Schema = Record<string, any>;
9
15
  export type MiddlewareOperation = "create" | "update" | "read" | "delete";
@@ -39,9 +45,18 @@ export declare class ConcurrencyError extends Error {
39
45
  readonly path: string;
40
46
  constructor(path: string);
41
47
  }
48
+ export declare class RateLimitError extends Error {
49
+ readonly retryAfter?: number | undefined;
50
+ constructor(retryAfter?: number | undefined);
51
+ }
52
+ export declare class CorruptedContentError extends Error {
53
+ readonly path: string;
54
+ constructor(path: string, cause: Error);
55
+ }
42
56
  export interface StorageResponse<T> {
43
57
  data: T;
44
58
  sha: string;
59
+ etag?: string;
45
60
  }
46
61
  export interface Validator<T> {
47
62
  validate: (data: unknown) => Promise<T> | T;
@@ -60,7 +75,7 @@ export interface IStorageProvider {
60
75
  listDirectory(path: string): Promise<{
61
76
  path: string;
62
77
  sha: string;
63
- type: "file" | "dir";
78
+ type: "file" | "dir" | "symlink" | "submodule";
64
79
  }[]>;
65
80
  commit(changes: CommitChange[], message: string): Promise<string>;
66
81
  }
@@ -6,3 +6,20 @@ export class ConcurrencyError extends Error {
6
6
  this.name = "ConcurrencyError";
7
7
  }
8
8
  }
9
+ export class RateLimitError extends Error {
10
+ retryAfter;
11
+ constructor(retryAfter) {
12
+ super(`GitHub API rate limit exceeded${retryAfter ? `. Retry after ${retryAfter}s` : ""}`);
13
+ this.retryAfter = retryAfter;
14
+ this.name = "RateLimitError";
15
+ }
16
+ }
17
+ export class CorruptedContentError extends Error {
18
+ path;
19
+ constructor(path, cause) {
20
+ super(`Corrupted or malformed content at ${path}: ${cause.message}`);
21
+ this.path = path;
22
+ this.name = "CorruptedContentError";
23
+ this.cause = cause;
24
+ }
25
+ }
@@ -0,0 +1,3 @@
1
+ export declare function validateName(name: string, context?: string): void;
2
+ export declare function validateId(id: string): void;
3
+ export declare function validatePath(path: string, context?: string): void;
@@ -0,0 +1,35 @@
1
+ const VALID_NAME_RE = /^[A-Za-z0-9._-]+$/;
2
+ export function validateName(name, context = "name") {
3
+ if (!name || typeof name !== "string") {
4
+ throw new Error(`Invalid ${context}: must be a non-empty string`);
5
+ }
6
+ if (name === "." || name === "..") {
7
+ throw new Error(`Invalid ${context}: "${name}" is a reserved name`);
8
+ }
9
+ if (name.includes("/") || name.includes("\\") || name.includes("\0")) {
10
+ throw new Error(`Invalid ${context}: "${name}" contains path separators or control characters`);
11
+ }
12
+ if (!VALID_NAME_RE.test(name)) {
13
+ throw new Error(`Invalid ${context}: "${name}" contains invalid characters`);
14
+ }
15
+ if (name.length > 255) {
16
+ throw new Error(`Invalid ${context}: exceeds maximum length of 255 characters`);
17
+ }
18
+ }
19
+ export function validateId(id) {
20
+ validateName(id, "id");
21
+ }
22
+ export function validatePath(path, context = "path") {
23
+ if (!path || typeof path !== "string") {
24
+ throw new Error(`Invalid ${context}: must be a non-empty string`);
25
+ }
26
+ if (path.startsWith("/")) {
27
+ throw new Error(`Invalid ${context}: "${path}" must not start with /`);
28
+ }
29
+ if (path.includes("..")) {
30
+ throw new Error(`Invalid ${context}: "${path}" contains path traversal ("..")`);
31
+ }
32
+ if (path.includes("\0")) {
33
+ throw new Error(`Invalid ${context}: "${path}" contains null byte`);
34
+ }
35
+ }
package/dist/index.d.ts CHANGED
@@ -2,4 +2,4 @@ export { GitHubDB } from "./ui/github-db.js";
2
2
  export { Collection } from "./ui/collection.js";
3
3
  export { Transaction } from "./ui/transaction.js";
4
4
  export * from "./core/types.js";
5
- export declare const version = "1.2.0";
5
+ export declare const version = "2.0.0";
package/dist/index.js CHANGED
@@ -2,4 +2,4 @@ export { GitHubDB } from "./ui/github-db.js";
2
2
  export { Collection } from "./ui/collection.js";
3
3
  export { Transaction } from "./ui/transaction.js";
4
4
  export * from "./core/types.js";
5
- export const version = "1.2.0";
5
+ export const version = "2.0.0";
@@ -1,4 +1,4 @@
1
- import { GitHubDBConfig, IStorageProvider, StorageResponse, CommitChange } from "../core/types.js";
1
+ import { RetryConfig, GitHubDBConfig, IStorageProvider, StorageResponse, CommitChange } from "../core/types.js";
2
2
  import { ICacheProvider } from "./cache-provider.js";
3
3
  export declare class GitHubStorageProvider implements IStorageProvider {
4
4
  private config;
@@ -6,8 +6,18 @@ export declare class GitHubStorageProvider implements IStorageProvider {
6
6
  private cache;
7
7
  private staleCache;
8
8
  private readonly DEFAULT_TTL;
9
+ private readonly MAX_STALE_ENTRIES;
9
10
  constructor(config: GitHubDBConfig, cache?: ICacheProvider);
11
+ private setStaleCache;
12
+ private parseRetryAfter;
13
+ private isSecondaryRateLimit;
14
+ private retryWithBackoff;
10
15
  testConnection(): Promise<boolean>;
16
+ testConnectionDetailed(): Promise<{
17
+ ok: boolean;
18
+ reason?: "auth" | "not-found" | "rate-limit" | "network" | "unknown";
19
+ error?: Error;
20
+ }>;
11
21
  exists(path: string): Promise<boolean>;
12
22
  readJson<T>(path: string): Promise<StorageResponse<T>>;
13
23
  writeJson<T>(path: string, content: T, message: string, sha?: string): Promise<string>;
@@ -16,6 +26,16 @@ export declare class GitHubStorageProvider implements IStorageProvider {
16
26
  listDirectory(path: string): Promise<{
17
27
  path: string;
18
28
  sha: string;
19
- type: "file" | "dir";
29
+ type: "file" | "dir" | "symlink" | "submodule";
20
30
  }[]>;
31
+ toJSON(): {
32
+ config: {
33
+ accessToken: string;
34
+ owner: string;
35
+ repo: string;
36
+ branch?: string;
37
+ cacheTTL?: number;
38
+ retry?: RetryConfig | false;
39
+ };
40
+ };
21
41
  }
@@ -1,12 +1,14 @@
1
1
  import { Octokit } from "@octokit/rest";
2
- import { ConcurrencyError, } from "../core/types.js";
2
+ import { ConcurrencyError, CorruptedContentError, RateLimitError, } from "../core/types.js";
3
3
  import { MemoryCacheProvider } from "./cache-provider.js";
4
+ import { validatePath } from "../core/validation.js";
4
5
  export class GitHubStorageProvider {
5
6
  config;
6
7
  octokit;
7
8
  cache;
8
9
  staleCache = new Map();
9
10
  DEFAULT_TTL = 0; // Default to 0 for consistency
11
+ MAX_STALE_ENTRIES = 1000;
10
12
  constructor(config, cache) {
11
13
  this.config = config;
12
14
  this.octokit = new Octokit({
@@ -14,28 +16,148 @@ export class GitHubStorageProvider {
14
16
  });
15
17
  this.cache = cache || new MemoryCacheProvider();
16
18
  }
19
+ setStaleCache(path, entry) {
20
+ if (this.staleCache.size >= this.MAX_STALE_ENTRIES) {
21
+ const oldest = this.staleCache.keys().next().value;
22
+ if (oldest)
23
+ this.staleCache.delete(oldest);
24
+ }
25
+ this.staleCache.set(path, entry);
26
+ }
27
+ parseRetryAfter(value) {
28
+ const num = Number(value);
29
+ if (Number.isFinite(num) && num > 0) {
30
+ return num * 1000;
31
+ }
32
+ const date = Date.parse(value);
33
+ if (Number.isFinite(date)) {
34
+ return Math.max(0, date - Date.now());
35
+ }
36
+ return NaN;
37
+ }
38
+ isSecondaryRateLimit(error) {
39
+ const headers = error?.response?.headers || {};
40
+ if (headers["retry-after"])
41
+ return true;
42
+ const message = error?.message || "";
43
+ if (message.includes("secondary rate limit") ||
44
+ message.includes("abuse detection") ||
45
+ message.includes("abuse mechanism")) {
46
+ return true;
47
+ }
48
+ return false;
49
+ }
50
+ async retryWithBackoff(fn) {
51
+ if (this.config.retry === false)
52
+ return fn();
53
+ const { maxRetries = 3, baseDelay = 1000, maxDelay = 10000, } = this.config.retry ?? {};
54
+ let lastError;
55
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
56
+ try {
57
+ return await fn();
58
+ }
59
+ catch (error) {
60
+ lastError = error;
61
+ const status = error?.status;
62
+ // Non-retryable: 409 (concurrency), 4xx client errors (except 429)
63
+ if (status === 409)
64
+ throw error;
65
+ if (status && status < 500 && status !== 429) {
66
+ if (status === 403 && this.isSecondaryRateLimit(error)) {
67
+ // Fall through to retry logic below
68
+ }
69
+ else {
70
+ throw error;
71
+ }
72
+ }
73
+ // Retry status-less errors that appear to be network errors
74
+ if (!status) {
75
+ const code = error?.code;
76
+ const isNetworkError = code === "ENOTFOUND" ||
77
+ code === "ECONNRESET" ||
78
+ code === "ETIMEDOUT" ||
79
+ code === "EAI_AGAIN" ||
80
+ code === "ECONNREFUSED" ||
81
+ code === "EPIPE" ||
82
+ !error?.response;
83
+ if (!isNetworkError) {
84
+ throw error;
85
+ }
86
+ // Fall through to retry logic below
87
+ }
88
+ // Last attempt — don't wait, just break
89
+ if (attempt === maxRetries)
90
+ break;
91
+ // Calculate delay
92
+ let delay;
93
+ if (status === 429 && error.response?.headers?.["retry-after"]) {
94
+ const retryAfter = error.response.headers["retry-after"];
95
+ delay = this.parseRetryAfter(retryAfter);
96
+ if (!Number.isFinite(delay) || delay < 0) {
97
+ delay = baseDelay * Math.pow(2, attempt);
98
+ }
99
+ }
100
+ else {
101
+ delay = baseDelay * Math.pow(2, attempt);
102
+ // Add jitter to avoid thundering herd on simultaneous retries
103
+ delay = delay * (0.5 + Math.random() * 0.5);
104
+ }
105
+ // Cap all delays (including Retry-After) against maxDelay
106
+ delay = Math.min(delay, maxDelay);
107
+ await new Promise((resolve) => setTimeout(resolve, delay));
108
+ }
109
+ }
110
+ // Exhausted retries on 429 → RateLimitError
111
+ if (lastError.status === 429) {
112
+ const retryAfter = lastError.response?.headers?.["retry-after"];
113
+ throw new RateLimitError(retryAfter ? parseInt(retryAfter, 10) : undefined);
114
+ }
115
+ throw lastError;
116
+ }
17
117
  async testConnection() {
18
118
  try {
19
- await this.octokit.repos.get({
119
+ await this.retryWithBackoff(() => this.octokit.repos.get({
20
120
  owner: this.config.owner,
21
121
  repo: this.config.repo,
22
- });
122
+ }));
23
123
  return true;
24
124
  }
25
125
  catch (error) {
26
126
  return false;
27
127
  }
28
128
  }
129
+ async testConnectionDetailed() {
130
+ try {
131
+ await this.retryWithBackoff(() => this.octokit.repos.get({
132
+ owner: this.config.owner,
133
+ repo: this.config.repo,
134
+ }));
135
+ return { ok: true };
136
+ }
137
+ catch (error) {
138
+ const status = error?.status;
139
+ if (status === 401)
140
+ return { ok: false, reason: "auth", error };
141
+ if (status === 404)
142
+ return { ok: false, reason: "not-found", error };
143
+ if (status === 429)
144
+ return { ok: false, reason: "rate-limit", error };
145
+ if (!status || error?.code === "ENOTFOUND" || error?.code === "ECONNRESET" || error?.code === "ETIMEDOUT") {
146
+ return { ok: false, reason: "network", error };
147
+ }
148
+ return { ok: false, reason: "unknown", error };
149
+ }
150
+ }
29
151
  async exists(path) {
30
- if (this.cache.get(path) || this.staleCache.has(path)) {
152
+ if (this.cache.get(path)) {
31
153
  return true;
32
154
  }
33
155
  try {
34
- await this.octokit.repos.getContent({
156
+ await this.retryWithBackoff(() => this.octokit.repos.getContent({
35
157
  owner: this.config.owner,
36
158
  repo: this.config.repo,
37
159
  path,
38
- });
160
+ }));
39
161
  return true;
40
162
  }
41
163
  catch (error) {
@@ -54,33 +176,52 @@ export class GitHubStorageProvider {
54
176
  // 2. Check stale cache for conditional request
55
177
  const stale = this.staleCache.get(path);
56
178
  try {
57
- const response = await this.octokit.repos.getContent({
179
+ const response = await this.retryWithBackoff(() => this.octokit.repos.getContent({
58
180
  owner: this.config.owner,
59
181
  repo: this.config.repo,
60
182
  path,
61
- headers: stale ? { "if-none-match": `"${stale.sha}"` } : {},
62
- });
183
+ headers: stale
184
+ ? {
185
+ "if-none-match": stale.etag || `"${stale.sha}"`,
186
+ }
187
+ : {},
188
+ }));
63
189
  if (Array.isArray(response.data)) {
64
190
  throw new Error("Path is a directory, not a file");
65
191
  }
66
192
  if (!("content" in response.data) || !("sha" in response.data)) {
67
193
  throw new Error("No content or SHA in response");
68
194
  }
69
- const binary = atob(response.data.content.replace(/\s/g, ""));
70
- const bytes = new Uint8Array(binary.length);
71
- for (let i = 0; i < binary.length; i++) {
72
- bytes[i] = binary.charCodeAt(i);
195
+ let content;
196
+ try {
197
+ const binary = atob(response.data.content.replace(/\s/g, ""));
198
+ const bytes = new Uint8Array(binary.length);
199
+ for (let i = 0; i < binary.length; i++) {
200
+ bytes[i] = binary.charCodeAt(i);
201
+ }
202
+ const decoder = new TextDecoder("utf-8");
203
+ content = decoder.decode(bytes);
204
+ }
205
+ catch (e) {
206
+ throw new CorruptedContentError(path, e);
207
+ }
208
+ let parsed;
209
+ try {
210
+ parsed = JSON.parse(content);
73
211
  }
74
- const decoder = new TextDecoder("utf-8");
75
- const content = decoder.decode(bytes);
212
+ catch (e) {
213
+ throw new CorruptedContentError(path, e);
214
+ }
215
+ const etag = response.headers?.etag || undefined;
76
216
  const result = {
77
- data: JSON.parse(content),
217
+ data: parsed,
78
218
  sha: response.data.sha,
219
+ etag,
79
220
  };
80
221
  // Update both caches
81
222
  const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
82
223
  this.cache.set(path, result, ttl);
83
- this.staleCache.set(path, result);
224
+ this.setStaleCache(path, result);
84
225
  return result;
85
226
  }
86
227
  catch (error) {
@@ -102,11 +243,11 @@ export class GitHubStorageProvider {
102
243
  }
103
244
  else {
104
245
  try {
105
- const existing = await this.octokit.repos.getContent({
246
+ const existing = await this.retryWithBackoff(() => this.octokit.repos.getContent({
106
247
  owner: this.config.owner,
107
248
  repo: this.config.repo,
108
249
  path,
109
- });
250
+ }));
110
251
  if (!Array.isArray(existing.data) && "sha" in existing.data) {
111
252
  internalSha = existing.data.sha;
112
253
  }
@@ -118,8 +259,11 @@ export class GitHubStorageProvider {
118
259
  }
119
260
  }
120
261
  }
262
+ // Note: internalSha is resolved before the retry wrapper. If a write fails
263
+ // with a 5xx after GitHub has already committed the file (e.g. network timeout),
264
+ // the retry will reuse the stale sha and may fail with a 409.
121
265
  try {
122
- const response = await this.octokit.repos.createOrUpdateFileContents({
266
+ const response = await this.retryWithBackoff(() => this.octokit.repos.createOrUpdateFileContents({
123
267
  owner: this.config.owner,
124
268
  repo: this.config.repo,
125
269
  path,
@@ -128,12 +272,15 @@ export class GitHubStorageProvider {
128
272
  .map((b) => String.fromCharCode(b))
129
273
  .join("")),
130
274
  sha: internalSha,
131
- });
132
- const newSha = response.data.content?.sha || "";
275
+ }));
276
+ const newSha = response.data.content?.sha;
277
+ if (!newSha) {
278
+ throw new Error(`writeJson succeeded but returned no SHA for ${path}`);
279
+ }
133
280
  const result = { data: content, sha: newSha };
134
281
  const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
135
282
  this.cache.set(path, result, ttl);
136
- this.staleCache.set(path, result);
283
+ this.setStaleCache(path, result);
137
284
  return newSha;
138
285
  }
139
286
  catch (error) {
@@ -147,21 +294,24 @@ export class GitHubStorageProvider {
147
294
  if (changes.length === 0) {
148
295
  throw new Error("No changes to commit");
149
296
  }
297
+ for (const change of changes) {
298
+ validatePath(change.path, "commit path");
299
+ }
150
300
  const branch = this.config.branch || "main";
151
301
  try {
152
302
  // 1. Get the current head SHA
153
- const { data: ref } = await this.octokit.git.getRef({
303
+ const { data: ref } = await this.retryWithBackoff(() => this.octokit.git.getRef({
154
304
  owner: this.config.owner,
155
305
  repo: this.config.repo,
156
306
  ref: `heads/${branch}`,
157
- });
307
+ }));
158
308
  const latestCommitSha = ref.object.sha;
159
309
  // 2. Get the tree SHA of the latest commit
160
- const { data: latestCommit } = await this.octokit.git.getCommit({
310
+ const { data: latestCommit } = await this.retryWithBackoff(() => this.octokit.git.getCommit({
161
311
  owner: this.config.owner,
162
312
  repo: this.config.repo,
163
313
  commit_sha: latestCommitSha,
164
- });
314
+ }));
165
315
  const baseTreeSha = latestCommit.tree.sha;
166
316
  // 3. Create a new tree with multiple files
167
317
  const treeEntries = changes.map((change) => {
@@ -180,27 +330,27 @@ export class GitHubStorageProvider {
180
330
  content: JSON.stringify(change.content, null, 2),
181
331
  };
182
332
  });
183
- const { data: newTree } = await this.octokit.git.createTree({
333
+ const { data: newTree } = await this.retryWithBackoff(() => this.octokit.git.createTree({
184
334
  owner: this.config.owner,
185
335
  repo: this.config.repo,
186
336
  base_tree: baseTreeSha,
187
337
  tree: treeEntries,
188
- });
338
+ }));
189
339
  // 4. Create a new commit
190
- const { data: newCommit } = await this.octokit.git.createCommit({
340
+ const { data: newCommit } = await this.retryWithBackoff(() => this.octokit.git.createCommit({
191
341
  owner: this.config.owner,
192
342
  repo: this.config.repo,
193
343
  message,
194
344
  tree: newTree.sha,
195
345
  parents: [latestCommitSha],
196
- });
346
+ }));
197
347
  // 5. Update the reference
198
- await this.octokit.git.updateRef({
348
+ await this.retryWithBackoff(() => this.octokit.git.updateRef({
199
349
  owner: this.config.owner,
200
350
  repo: this.config.repo,
201
351
  ref: `heads/${branch}`,
202
352
  sha: newCommit.sha,
203
- });
353
+ }));
204
354
  // 6. Update cache for all involved files
205
355
  const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
206
356
  for (const entry of newTree.tree) {
@@ -209,37 +359,51 @@ export class GitHubStorageProvider {
209
359
  if (change) {
210
360
  const result = { data: change.content, sha: entry.sha };
211
361
  this.cache.set(entry.path, result, ttl);
212
- this.staleCache.set(entry.path, result);
362
+ this.setStaleCache(entry.path, result);
213
363
  }
214
364
  }
215
365
  }
366
+ for (const change of changes) {
367
+ if (change.content === null) {
368
+ this.cache.delete(change.path);
369
+ this.staleCache.delete(change.path);
370
+ }
371
+ }
216
372
  return newCommit.sha;
217
373
  }
218
374
  catch (error) {
219
- if (error.status === 409) {
375
+ if (error.status === 409 || error.status === 422) {
220
376
  throw new ConcurrencyError("batch-commit");
221
377
  }
222
378
  throw error;
223
379
  }
224
380
  }
225
381
  async deleteFile(path, message, sha) {
226
- await this.octokit.repos.deleteFile({
227
- owner: this.config.owner,
228
- repo: this.config.repo,
229
- path,
230
- message,
231
- sha,
232
- });
382
+ try {
383
+ await this.retryWithBackoff(() => this.octokit.repos.deleteFile({
384
+ owner: this.config.owner,
385
+ repo: this.config.repo,
386
+ path,
387
+ message,
388
+ sha,
389
+ }));
390
+ }
391
+ catch (error) {
392
+ if (error.status === 409) {
393
+ throw new ConcurrencyError(path);
394
+ }
395
+ throw error;
396
+ }
233
397
  this.cache.delete(path);
234
398
  this.staleCache.delete(path);
235
399
  }
236
400
  async listDirectory(path) {
237
401
  try {
238
- const response = await this.octokit.repos.getContent({
402
+ const response = await this.retryWithBackoff(() => this.octokit.repos.getContent({
239
403
  owner: this.config.owner,
240
404
  repo: this.config.repo,
241
405
  path,
242
- });
406
+ }));
243
407
  if (!Array.isArray(response.data)) {
244
408
  throw new Error("Path is not a directory");
245
409
  }
@@ -256,4 +420,12 @@ export class GitHubStorageProvider {
256
420
  throw error;
257
421
  }
258
422
  }
423
+ toJSON() {
424
+ return {
425
+ config: {
426
+ ...this.config,
427
+ accessToken: "[redacted]",
428
+ },
429
+ };
430
+ }
259
431
  }
@@ -15,7 +15,7 @@ export declare class TransactionStorageProvider implements IStorageProvider {
15
15
  listDirectory(path: string): Promise<{
16
16
  path: string;
17
17
  sha: string;
18
- type: "file" | "dir";
18
+ type: "file" | "dir" | "symlink" | "submodule";
19
19
  }[]>;
20
20
  commit(changes: CommitChange[], message: string): Promise<string>;
21
21
  /**
@@ -13,13 +13,18 @@ export class TransactionStorageProvider {
13
13
  }
14
14
  async exists(path) {
15
15
  if (this.pendingChanges.has(path)) {
16
- return true;
16
+ return this.pendingChanges.get(path) !== null;
17
17
  }
18
18
  return this.baseStorage.exists(path);
19
19
  }
20
20
  async readJson(path) {
21
21
  const pending = this.pendingChanges.get(path);
22
22
  if (pending !== undefined) {
23
+ if (pending === null) {
24
+ throw Object.assign(new Error(`File not found in transaction: ${path}`), {
25
+ status: 404,
26
+ });
27
+ }
23
28
  return {
24
29
  data: pending,
25
30
  sha: "transaction-pending-sha",
@@ -35,9 +40,30 @@ export class TransactionStorageProvider {
35
40
  this.pendingChanges.set(path, null);
36
41
  }
37
42
  async listDirectory(path) {
38
- // For now, we delegate to base storage.
39
- // In sharded mode, we might need to merge with pending changes.
40
- return this.baseStorage.listDirectory(path);
43
+ const baseList = await this.baseStorage.listDirectory(path);
44
+ const basePaths = new Set(baseList.map((e) => e.path));
45
+ const merged = [...baseList];
46
+ for (const [pendingPath, content] of this.pendingChanges) {
47
+ if (!pendingPath.startsWith(path))
48
+ continue;
49
+ if (content === null) {
50
+ // Deletion — remove from list
51
+ continue;
52
+ }
53
+ if (!basePaths.has(pendingPath)) {
54
+ // New file — add to list
55
+ merged.push({
56
+ path: pendingPath,
57
+ sha: "transaction-pending-sha",
58
+ type: "file",
59
+ });
60
+ }
61
+ }
62
+ // Filter out deleted paths
63
+ return merged.filter((entry) => {
64
+ const pending = this.pendingChanges.get(entry.path);
65
+ return pending !== undefined ? pending !== null : true;
66
+ });
41
67
  }
42
68
  async commit(changes, message) {
43
69
  return this.baseStorage.commit(changes, message);
@@ -2,6 +2,7 @@ import enquirer from "enquirer";
2
2
  import chalk from "chalk";
3
3
  import { Octokit } from "@octokit/rest";
4
4
  import { GitHubStorageProvider } from "../../infrastructure/github-storage.js";
5
+ import { validateName } from "../../core/validation.js";
5
6
  export async function initCommand() {
6
7
  console.log(chalk.yellow("\n--- Database Initialization ---\n"));
7
8
  try {
@@ -29,14 +30,21 @@ export async function initCommand() {
29
30
  accessToken: answers.accessToken,
30
31
  });
31
32
  process.stdout.write(chalk.blue("Verifying connection... "));
32
- const success = await storage.testConnection();
33
- if (success) {
33
+ const result = await storage.testConnectionDetailed();
34
+ if (result.ok) {
34
35
  console.log(chalk.green("Success!"));
35
36
  console.log(chalk.dim("\nYou can now use these credentials in your application configuration."));
36
37
  }
37
38
  else {
38
- console.log(chalk.red("Failed."));
39
- console.log(chalk.red("Please check your token permissions and repository details."));
39
+ console.log(chalk.red(`Failed — ${result.reason || "unknown error"}`));
40
+ if (result.reason === "auth")
41
+ console.log(chalk.red("Check your token and permissions."));
42
+ else if (result.reason === "not-found")
43
+ console.log(chalk.red("Check that the owner and repo are correct."));
44
+ else if (result.reason === "rate-limit")
45
+ console.log(chalk.red("You've been rate limited. Try again later."));
46
+ else
47
+ console.log(chalk.red("Please check your token permissions and repository details."));
40
48
  }
41
49
  }
42
50
  catch (error) {
@@ -52,6 +60,8 @@ export async function listCollectionsCommand() {
52
60
  console.log(chalk.red("Error: Environment variables GH_DB_OWNER, GH_DB_REPO, and GH_DB_TOKEN must be set."));
53
61
  return;
54
62
  }
63
+ validateName(owner, "owner");
64
+ validateName(repo, "repo");
55
65
  const octokit = new Octokit({ auth });
56
66
  try {
57
67
  const { data } = await octokit.repos.getContent({
@@ -60,9 +70,13 @@ export async function listCollectionsCommand() {
60
70
  path: "",
61
71
  });
62
72
  if (Array.isArray(data)) {
63
- const collections = data
64
- .filter((item) => item.name.endsWith(".json"))
73
+ const jsonCollections = data
74
+ .filter((item) => item.type === "file" && item.name.endsWith(".json"))
65
75
  .map((item) => item.name.replace(".json", ""));
76
+ const dirCollections = data
77
+ .filter((item) => item.type === "dir")
78
+ .map((item) => `${item.name}/ (sharded)`);
79
+ const collections = [...jsonCollections, ...dirCollections];
66
80
  if (collections.length === 0) {
67
81
  console.log(chalk.dim("No collections found in the repository."));
68
82
  }
@@ -74,7 +88,7 @@ export async function listCollectionsCommand() {
74
88
  }
75
89
  }
76
90
  catch (error) {
77
- console.error(chalk.red(`Error fetching collections: ${error.message}`));
91
+ console.error(chalk.red(`Error fetching collections: GitHub API error`));
78
92
  }
79
93
  }
80
94
  export async function inspectCollectionCommand(name) {
@@ -86,23 +100,37 @@ export async function inspectCollectionCommand(name) {
86
100
  console.log(chalk.red("Error: Environment variables GH_DB_OWNER, GH_DB_REPO, and GH_DB_TOKEN must be set."));
87
101
  return;
88
102
  }
103
+ validateName(name, "collection name");
89
104
  const storage = new GitHubStorageProvider({
90
105
  owner,
91
106
  repo,
92
107
  accessToken: auth,
93
108
  });
94
109
  try {
95
- const path = `${name}.json`;
96
- if (!(await storage.exists(path))) {
110
+ const singlePath = `${name}.json`;
111
+ const shardedPath = `${name}`;
112
+ let isSingleFile = await storage.exists(singlePath);
113
+ let isSharded = !isSingleFile ? await storage.exists(shardedPath) : false;
114
+ if (!isSingleFile && !isSharded) {
97
115
  console.log(chalk.red(`Collection "${name}" does not exist.`));
98
116
  return;
99
117
  }
100
- const response = await storage.readJson(path);
101
- console.log(JSON.stringify(response.data, null, 2));
102
- console.log(chalk.dim(`\nTotal records: ${response.data.length}`));
103
- console.log(chalk.dim(`Current SHA: ${response.sha}`));
118
+ if (isSingleFile) {
119
+ const response = await storage.readJson(singlePath);
120
+ console.log(JSON.stringify(response.data, null, 2));
121
+ console.log(chalk.dim(`\nTotal records: ${response.data.length}`));
122
+ console.log(chalk.dim(`Current SHA: ${response.sha}`));
123
+ }
124
+ else {
125
+ const files = await storage.listDirectory(shardedPath);
126
+ const jsonFiles = files.filter((f) => f.type === "file" && f.path.endsWith(".json"));
127
+ console.log(chalk.dim(`\nSharded collection with ${jsonFiles.length} documents:`));
128
+ for (const file of jsonFiles) {
129
+ console.log(chalk.cyan(` • ${file.path}`));
130
+ }
131
+ }
104
132
  }
105
133
  catch (error) {
106
- console.error(chalk.red(`Error inspecting collection: ${error.message}`));
134
+ console.error(chalk.red(`Error inspecting collection: GitHub API error`));
107
135
  }
108
136
  }
@@ -13,7 +13,7 @@ console.log(boxen(chalk.bold.blue("gh-as-db CLI"), {
13
13
  program
14
14
  .name("gh-as-db")
15
15
  .description("CLI to manage your GitHub-based database")
16
- .version("0.1.0");
16
+ .version("2.0.0");
17
17
  program
18
18
  .command("init")
19
19
  .description("Initialize connection and verify repository")
@@ -1,5 +1,6 @@
1
1
  import { ConcurrencyError, } from "../core/types.js";
2
2
  import { Indexer } from "../core/indexer.js";
3
+ import { validateName, validateId } from "../core/validation.js";
3
4
  export class Collection {
4
5
  name;
5
6
  storage;
@@ -14,6 +15,7 @@ export class Collection {
14
15
  constructor(name, storage, middlewareOrOptions) {
15
16
  this.name = name;
16
17
  this.storage = storage;
18
+ validateName(name);
17
19
  if (Array.isArray(middlewareOrOptions)) {
18
20
  this.middleware = middlewareOrOptions;
19
21
  this.strategy = "single-file";
@@ -28,6 +30,7 @@ export class Collection {
28
30
  return this.strategy === "sharded" ? `${this.name}/` : `${this.name}.json`;
29
31
  }
30
32
  getItemPath(id) {
33
+ validateId(id);
31
34
  return this.strategy === "sharded"
32
35
  ? `${this.name}/${id}.json`
33
36
  : `${this.name}.json`;
@@ -49,18 +52,21 @@ export class Collection {
49
52
  }
50
53
  }
51
54
  if (this.strategy === "sharded") {
55
+ validateId(finalItem.id);
52
56
  const itemPath = this.getItemPath(finalItem.id);
53
57
  const sha = await this.storage.writeJson(itemPath, finalItem, `Create item ${finalItem.id} in ${this.name}`, this.shas.get(itemPath));
54
58
  this.shas.set(itemPath, sha);
55
59
  if (this.dataLoaded) {
56
60
  const index = this.items.findIndex((i) => i.id === finalItem.id);
57
61
  if (index > -1) {
62
+ const oldItem = this.items[index];
58
63
  this.items[index] = finalItem;
64
+ this.indexer.update(oldItem, finalItem);
59
65
  }
60
66
  else {
61
67
  this.items.push(finalItem);
68
+ this.indexer.add(finalItem);
62
69
  }
63
- this.indexer.add(finalItem);
64
70
  }
65
71
  return finalItem;
66
72
  }
@@ -77,7 +83,8 @@ export class Collection {
77
83
  }
78
84
  }
79
85
  catch (error) {
80
- // If file doesn't exist, start with empty array
86
+ if (error?.status !== 404)
87
+ throw error;
81
88
  }
82
89
  }
83
90
  items.push(finalItem);
@@ -115,19 +122,38 @@ export class Collection {
115
122
  queryOrPredicate.filters[0].operator === "eq" &&
116
123
  this.indexer.hasIndex(queryOrPredicate.filters[0].field)) {
117
124
  const filter = queryOrPredicate.filters[0];
118
- const results = this.indexer.query(filter.field, filter.value);
119
- if (results !== null) {
125
+ const rawResults = this.indexer.query(filter.field, filter.value);
126
+ if (rawResults !== null) {
127
+ let results = (await Promise.all(rawResults.map(async (item) => {
128
+ let current = item;
129
+ for (const mw of this.middleware) {
130
+ if (mw.afterRead)
131
+ current = await mw.afterRead(current, context);
132
+ }
133
+ return current;
134
+ })));
135
+ if (queryOrPredicate) {
136
+ results = this.applyQueryOptions(results, queryOrPredicate);
137
+ }
120
138
  return results;
121
139
  }
122
140
  }
123
141
  // Fallback to full scan of current in-memory items
142
+ let cachedItems = (await Promise.all(this.items.map(async (item) => {
143
+ let current = item;
144
+ for (const mw of this.middleware) {
145
+ if (mw.afterRead)
146
+ current = await mw.afterRead(current, context);
147
+ }
148
+ return current;
149
+ })));
124
150
  if (typeof queryOrPredicate === "function") {
125
- return this.items.filter(queryOrPredicate);
151
+ return cachedItems.filter(queryOrPredicate);
126
152
  }
127
153
  if (queryOrPredicate) {
128
- return this.applyQueryOptions(this.items, queryOrPredicate);
154
+ return this.applyQueryOptions(cachedItems, queryOrPredicate);
129
155
  }
130
- return this.items;
156
+ return cachedItems;
131
157
  }
132
158
  if (this.strategy === "sharded") {
133
159
  const files = await this.storage.listDirectory(this.name);
@@ -144,6 +170,7 @@ export class Collection {
144
170
  this.lastSha = response.sha;
145
171
  items = response.data;
146
172
  }
173
+ const rawItems = items;
147
174
  items = await Promise.all(items.map(async (item) => {
148
175
  let currentItem = item;
149
176
  for (const mw of this.middleware) {
@@ -153,10 +180,10 @@ export class Collection {
153
180
  }
154
181
  return currentItem;
155
182
  }));
156
- // Build index and store in-memory items
183
+ // Build index and store RAW items (before afterRead) in memory
157
184
  if (!this.dataLoaded) {
158
- this.items = items;
159
- this.indexer.build(items, items.length > 0 ? Object.keys(items[0]) : []);
185
+ this.items = rawItems;
186
+ this.indexer.build(rawItems, rawItems.length > 0 ? Object.keys(rawItems[0]) : []);
160
187
  this.dataLoaded = true;
161
188
  }
162
189
  if (typeof queryOrPredicate === "function") {
@@ -169,10 +196,20 @@ export class Collection {
169
196
  }
170
197
  applyQueryOptions(items, options) {
171
198
  let result = [...items];
199
+ const PROTOTYPE_KEYS = new Set(["__proto__", "constructor", "prototype"]);
200
+ const VALID_OPERATORS = new Set([
201
+ "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in",
202
+ ]);
172
203
  // Apply filters
173
204
  if (options.filters) {
174
205
  for (const filter of options.filters) {
175
206
  result = result.filter((item) => {
207
+ if (PROTOTYPE_KEYS.has(filter.field)) {
208
+ return true;
209
+ }
210
+ if (!VALID_OPERATORS.has(filter.operator)) {
211
+ throw new Error(`Unknown filter operator: "${filter.operator}". Supported: ${[...VALID_OPERATORS].join(", ")}`);
212
+ }
176
213
  const val = item[filter.field];
177
214
  switch (filter.operator) {
178
215
  case "eq":
@@ -192,10 +229,12 @@ export class Collection {
192
229
  return val.includes(filter.value);
193
230
  }
194
231
  return (typeof val === "string" && val.includes(filter.value));
195
- case "in":
196
- return Array.isArray(filter.value) && filter.value.includes(val);
197
- default:
198
- return true;
232
+ case "in": {
233
+ if (!Array.isArray(filter.value)) {
234
+ throw new Error(`"in" operator requires an array value, got ${typeof filter.value}`);
235
+ }
236
+ return filter.value.includes(val);
237
+ }
199
238
  }
200
239
  });
201
240
  }
@@ -217,14 +256,33 @@ export class Collection {
217
256
  // Apply pagination
218
257
  if (options.pagination) {
219
258
  const { limit, offset = 0 } = options.pagination;
259
+ if (typeof offset !== "number" || offset < 0 || !Number.isFinite(offset)) {
260
+ throw new Error(`Invalid pagination offset: ${offset}`);
261
+ }
262
+ if (limit !== undefined && (typeof limit !== "number" || limit < 0 || !Number.isFinite(limit))) {
263
+ throw new Error(`Invalid pagination limit: ${limit}`);
264
+ }
220
265
  result = result.slice(offset, limit ? offset + limit : undefined);
221
266
  }
222
267
  return result;
223
268
  }
224
269
  async findById(id) {
270
+ validateId(id);
225
271
  if (this.dataLoaded && this.indexer.hasIndex("id")) {
226
272
  const results = this.indexer.query("id", id);
227
- return results && results.length > 0 ? results[0] : null;
273
+ if (results && results.length > 0) {
274
+ let item = results[0];
275
+ const context = {
276
+ collection: this.name,
277
+ operation: "read",
278
+ };
279
+ for (const mw of this.middleware) {
280
+ if (mw.afterRead)
281
+ item = await mw.afterRead(item, context);
282
+ }
283
+ return item;
284
+ }
285
+ return null;
228
286
  }
229
287
  if (this.strategy === "sharded") {
230
288
  try {
@@ -255,13 +313,20 @@ export class Collection {
255
313
  return items.find((item) => item.id === id) || null;
256
314
  }
257
315
  async update(id, updates) {
258
- const items = await this.find();
316
+ validateId(id);
317
+ const items = [...(await this.find())];
259
318
  const index = items.findIndex((item) => item.id === id);
260
319
  if (index === -1) {
261
320
  throw new Error(`Item with id ${id} not found in ${this.name}`);
262
321
  }
263
322
  const originalItem = items[index];
264
- items[index] = { ...items[index], ...updates };
323
+ const safeUpdates = {};
324
+ for (const key of Object.keys(updates)) {
325
+ if (key !== "__proto__" && key !== "constructor" && key !== "prototype") {
326
+ safeUpdates[key] = updates[key];
327
+ }
328
+ }
329
+ items[index] = { ...items[index], ...safeUpdates };
265
330
  let finalItem = items[index];
266
331
  // 1. Validation
267
332
  if (this.validator) {
@@ -278,7 +343,11 @@ export class Collection {
278
343
  }
279
344
  }
280
345
  items[index] = finalItem;
346
+ validateId(id);
281
347
  if (this.strategy === "sharded") {
348
+ if (updates.id !== undefined && updates.id !== id) {
349
+ throw new Error(`Cannot change id of a sharded item. Delete and re-create instead.`);
350
+ }
282
351
  const itemPath = this.getItemPath(id);
283
352
  try {
284
353
  const sha = await this.storage.writeJson(itemPath, finalItem, `Update item ${id} in ${this.name}`, this.shas.get(itemPath));
@@ -309,6 +378,7 @@ export class Collection {
309
378
  return items[index];
310
379
  }
311
380
  async delete(id) {
381
+ validateId(id);
312
382
  const items = await this.find();
313
383
  const index = items.findIndex((item) => item.id === id);
314
384
  if (index === -1) {
@@ -316,22 +386,40 @@ export class Collection {
316
386
  }
317
387
  const itemToDelete = items[index];
318
388
  const filtered = items.filter((_, i) => i !== index);
389
+ validateId(id);
319
390
  if (this.strategy === "sharded") {
320
391
  const itemPath = this.getItemPath(id);
321
- const sha = this.shas.get(itemPath);
322
- if (!sha) {
323
- // We need the SHA to delete. If we don't have it, we must fetch it.
324
- const response = await this.storage.readJson(itemPath);
325
- await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, response.sha);
392
+ try {
393
+ const sha = this.shas.get(itemPath);
394
+ if (!sha) {
395
+ const response = await this.storage.readJson(itemPath);
396
+ await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, response.sha);
397
+ }
398
+ else {
399
+ await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, sha);
400
+ }
401
+ this.shas.delete(itemPath);
326
402
  }
327
- else {
328
- await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, sha);
403
+ catch (error) {
404
+ if (error instanceof ConcurrencyError) {
405
+ this.dataLoaded = false;
406
+ this.shas.delete(itemPath);
407
+ }
408
+ throw error;
329
409
  }
330
- this.shas.delete(itemPath);
331
410
  }
332
411
  else {
333
412
  try {
334
- this.lastSha = await this.storage.writeJson(this.path, filtered, `Delete item ${id} from ${this.name}`, this.lastSha);
413
+ if (filtered.length === 0) {
414
+ const fileExists = await this.storage.exists(this.path);
415
+ if (fileExists) {
416
+ const existing = await this.storage.readJson(this.path);
417
+ await this.storage.deleteFile(this.path, `Delete item ${id} from ${this.name}`, existing.sha);
418
+ }
419
+ }
420
+ else {
421
+ this.lastSha = await this.storage.writeJson(this.path, filtered, `Delete item ${id} from ${this.name}`, this.lastSha);
422
+ }
335
423
  }
336
424
  catch (error) {
337
425
  if (error instanceof ConcurrencyError) {
@@ -1,4 +1,4 @@
1
- import { GitHubDBConfig, IStorageProvider, Middleware, Schema, Validator } from "../core/types.js";
1
+ import { GitHubDBConfig, IStorageProvider, Middleware, Schema, StorageStrategy, Validator } from "../core/types.js";
2
2
  import { Collection } from "./collection.js";
3
3
  import { Transaction } from "./transaction.js";
4
4
  export declare class GitHubDB {
@@ -9,6 +9,14 @@ export declare class GitHubDB {
9
9
  collection<T extends Schema>(name: string, options?: {
10
10
  middleware?: Middleware<T>[];
11
11
  validator?: Validator<T>;
12
+ strategy?: StorageStrategy;
12
13
  }): Collection<T>;
13
- transaction(fn: (tx: Transaction) => Promise<void>, message?: string): Promise<string>;
14
+ transaction(fn: (tx: Transaction) => Promise<void>, message?: string): Promise<string | null>;
15
+ toJSON(): {
16
+ config: {
17
+ owner: string;
18
+ repo: string;
19
+ accessToken: string;
20
+ };
21
+ };
14
22
  }
@@ -28,4 +28,13 @@ export class GitHubDB {
28
28
  await fn(tx);
29
29
  return tx.commit(message);
30
30
  }
31
+ toJSON() {
32
+ return {
33
+ config: {
34
+ owner: this.config.owner,
35
+ repo: this.config.repo,
36
+ accessToken: "[redacted]",
37
+ },
38
+ };
39
+ }
31
40
  }
@@ -19,5 +19,5 @@ export declare class Transaction {
19
19
  * Commits all buffered changes in the transaction to GitHub in a single commit.
20
20
  * @internal
21
21
  */
22
- commit(message: string): Promise<string>;
22
+ commit(message: string): Promise<string | null>;
23
23
  }
@@ -24,7 +24,7 @@ export class Transaction {
24
24
  async commit(message) {
25
25
  const changes = this.txStorage.getChanges();
26
26
  if (changes.length === 0) {
27
- return "";
27
+ return null;
28
28
  }
29
29
  return this.txStorage.commit(changes, message);
30
30
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "gh-as-db",
3
- "version": "1.2.0",
3
+ "version": "2.0.0",
4
4
  "description": "Use a private GitHub repository as a database for your application.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -18,14 +18,9 @@
18
18
  "files": [
19
19
  "dist",
20
20
  "README.md",
21
- "LICENSE"
21
+ "LICENSE",
22
+ "CHANGELOG.md"
22
23
  ],
23
- "scripts": {
24
- "test": "vitest",
25
- "test:run": "vitest run",
26
- "build": "tsc",
27
- "prepublishOnly": "npm run build"
28
- },
29
24
  "keywords": [
30
25
  "github",
31
26
  "database",
@@ -60,5 +55,10 @@
60
55
  "chalk": "^5.4.1",
61
56
  "commander": "^13.1.0",
62
57
  "enquirer": "^2.4.1"
58
+ },
59
+ "scripts": {
60
+ "test": "vitest",
61
+ "test:run": "vitest run",
62
+ "build": "tsc"
63
63
  }
64
- }
64
+ }