gh-as-db 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +58 -3
- package/dist/core/types.d.ts +16 -1
- package/dist/core/types.js +17 -0
- package/dist/core/validation.d.ts +3 -0
- package/dist/core/validation.js +35 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/infrastructure/github-storage.d.ts +22 -2
- package/dist/infrastructure/github-storage.js +216 -44
- package/dist/infrastructure/transaction-storage.d.ts +1 -1
- package/dist/infrastructure/transaction-storage.js +30 -4
- package/dist/ui/cli/commands.js +42 -14
- package/dist/ui/cli/index.js +1 -1
- package/dist/ui/collection.js +114 -26
- package/dist/ui/github-db.d.ts +10 -2
- package/dist/ui/github-db.js +9 -0
- package/dist/ui/transaction.d.ts +1 -1
- package/dist/ui/transaction.js +1 -1
- package/package.json +9 -9
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [2.0.0] — 2026-07-17
|
|
4
|
+
|
|
5
|
+
### Breaking Changes
|
|
6
|
+
|
|
7
|
+
- **`transaction()` return type**: `Promise<string>` → `Promise<string | null>`. Empty transactions now return `null` instead of `""`. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
8
|
+
- **Collection name validation**: Names must match `[A-Za-z0-9._-]` (max 255 chars). Names with spaces, slashes, or special characters now throw. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
9
|
+
- **Item ID validation**: IDs are validated on `create()`, `findById()`, `update()`, and `delete()`. IDs must match `[A-Za-z0-9._-]` (max 255 chars). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
10
|
+
- **Unknown filter operators now throw** instead of silently skipping. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
11
|
+
- **`"in"` filter operator requires an array value** — non-array values now throw. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
12
|
+
- **Invalid pagination throws**: negative/infinite `offset` or `limit` now throws. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
13
|
+
- **Sharded items: id is now immutable** — `update()` with a different id on sharded collections throws. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
14
|
+
- **`create()` no longer silently swallows transient read errors** — prevents data loss from overwriting entire collections. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- **Path traversal protection**: Collection names, item IDs, and commit paths validated against `../`, null bytes, and path separators. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
19
|
+
- **Prototype pollution hardening**: `__proto__`, `constructor`, and `prototype` keys rejected in filter fields and update payloads. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
20
|
+
- **Network error retry**: ENOTFOUND, ECONNRESET, ETIMEDOUT, ECONNREFUSED, EPIPE, and other network errors are retried with exponential backoff. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
21
|
+
- **Secondary rate limit detection**: 403 responses with `Retry-After` header or secondary rate limit message are retried. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
22
|
+
- **`Retry-After` HTTP-date parsing**: Falls back to exponential backoff on malformed values. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
23
|
+
- **Real ETag capture** for conditional `GET` requests (`If-None-Match`). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
24
|
+
- **`CorruptedContentError`**: Explicit error for malformed JSON or binary content. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
25
|
+
- **`toJSON()` on `GitHubDB` and `GitHubStorageProvider`** — accessToken redacted for safe logging. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
26
|
+
- **CLI: sharded collection detection** — `list` and `inspect` commands detect directory-based (sharded) collections. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
27
|
+
- **CLI: detailed connection test** — categorized failure reasons (auth, not-found, rate-limit, network). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
|
|
31
|
+
- **`afterRead` middleware now applies on all read paths** — including cached, indexed, and fast-path reads. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
32
|
+
- **Items stored as raw data** — `afterRead` transformations are no longer persisted to storage. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
33
|
+
- **`update()` works on immutable copies** — no longer mutates in-memory items in-place. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
34
|
+
- **Sharded upsert uses `indexer.update()`** (not bare `add()`) — fixes stale index entries. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
35
|
+
- **`deleteFile` wraps 409 → `ConcurrencyError`**; **`commit()` maps 422 → `ConcurrencyError`**. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
36
|
+
- **`exists()` only trusts fresh cache**; stale cache bounded to 1000 entries (LRU). ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
37
|
+
- **Transaction isolation**: `listDirectory` merges pending creates/deletes; `exists()` handles pending deletes. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
38
|
+
- **CLI error messages genericized** — no raw API errors leaked to terminal. ([#5](https://github.com/Almyk/gh-as-db/pull/5))
|
|
39
|
+
|
|
40
|
+
### Security
|
|
41
|
+
|
|
42
|
+
- Path traversal hardening (CWE-22)
|
|
43
|
+
- Prototype pollution hardening (CWE-1321)
|
|
44
|
+
- accessToken redaction in `toJSON()` / logging
|
|
45
|
+
- CLI error message sanitization
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## [1.3.0] — 2025-12-03
|
|
50
|
+
|
|
51
|
+
- Initial public release
|
package/README.md
CHANGED
|
@@ -11,6 +11,7 @@ Use a private GitHub repository as a database for your application. `gh-as-db` p
|
|
|
11
11
|
- 🔐 **Secure**: Designed for private repositories using Personal Access Tokens (PAT).
|
|
12
12
|
- 🚀 **Performance**: Built-in in-memory caching and auto-indexing for fast local queries.
|
|
13
13
|
- 🛡️ **Concurrency**: Optimistic locking using Git SHAs to prevent data loss.
|
|
14
|
+
- 🔄 **Retry & Rate Limits**: Automatic retries with exponential backoff for transient errors and GitHub rate limits.
|
|
14
15
|
- 🔗 **Transactions**: Group multiple operations into a single atomic Git commit.
|
|
15
16
|
- 📁 **Sharding**: One-file-per-document storage strategy for massive collections.
|
|
16
17
|
- 🧩 **Middleware**: Extensible hooks for data validation or transformation.
|
|
@@ -22,7 +23,7 @@ Use a private GitHub repository as a database for your application. `gh-as-db` p
|
|
|
22
23
|
## Installation
|
|
23
24
|
|
|
24
25
|
```bash
|
|
25
|
-
|
|
26
|
+
pnpm add gh-as-db
|
|
26
27
|
```
|
|
27
28
|
|
|
28
29
|
## Quick Start
|
|
@@ -43,6 +44,7 @@ const db = new GitHubDB({
|
|
|
43
44
|
});
|
|
44
45
|
|
|
45
46
|
// Access the 'users' collection
|
|
47
|
+
// Note: avoid logging or serializing the db instance — the access token is stored on db.config
|
|
46
48
|
const users = db.collection<User>('users');
|
|
47
49
|
|
|
48
50
|
// Create a new user
|
|
@@ -84,7 +86,8 @@ const db = new GitHubDB({
|
|
|
84
86
|
accessToken: string; // GitHub PAT with 'repo' scope
|
|
85
87
|
owner: string; // Repository owner
|
|
86
88
|
repo: string; // Repository name
|
|
87
|
-
cacheTTL?: number; // Optional: Cache TTL in ms. Default is 0 (strict consistency).
|
|
89
|
+
cacheTTL?: number; // Optional: Cache TTL in ms. Default is 0 (strict consistency — always validates with GitHub). Set to a positive value (e.g. 5000) to enable in-memory caching for performance. Note: cacheTTL > 0 may return stale data across instances until the TTL expires.
|
|
90
|
+
retry?: RetryConfig | false; // Optional: Retry config, or false to disable. See below.
|
|
88
91
|
});
|
|
89
92
|
```
|
|
90
93
|
|
|
@@ -193,6 +196,51 @@ const users = db.collection<User>('users', {
|
|
|
193
196
|
- 📈 **Scalability**: Avoids GitHub's file size limits and reduces merge conflicts.
|
|
194
197
|
- 🧹 **Cleanliness**: Better organization for repositories with thousands of documents.
|
|
195
198
|
|
|
199
|
+
**Important**: In sharded mode, item `id` values cannot be changed via `update()`. To change an id, delete the item and re-create it with the new id.
|
|
200
|
+
|
|
201
|
+
### Retry & Rate Limit Handling
|
|
202
|
+
|
|
203
|
+
All GitHub API calls are automatically retried on transient errors (429 rate limits, 500/502/503 server errors) with exponential backoff. If GitHub returns a `Retry-After` header, it is respected.
|
|
204
|
+
|
|
205
|
+
```typescript
|
|
206
|
+
const db = new GitHubDB({
|
|
207
|
+
accessToken: process.env.GITHUB_TOKEN,
|
|
208
|
+
owner: 'your-username',
|
|
209
|
+
repo: 'my-data-repo',
|
|
210
|
+
retry: {
|
|
211
|
+
maxRetries: 3, // Default: 3
|
|
212
|
+
baseDelay: 1000, // Default: 1000ms
|
|
213
|
+
maxDelay: 10000, // Default: 10000ms
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
To disable retries entirely:
|
|
219
|
+
|
|
220
|
+
```typescript
|
|
221
|
+
const db = new GitHubDB({
|
|
222
|
+
// ...
|
|
223
|
+
retry: false,
|
|
224
|
+
});
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
Non-transient errors (401, 404, and permission-related 403) are thrown immediately. GitHub secondary rate limits (abuse-detection 403 responses) are retried automatically with backoff, just like 429 rate limits. Concurrency conflicts (409) still throw `ConcurrencyError` immediately. If rate limit retries are exhausted, a `RateLimitError` is thrown.
|
|
228
|
+
|
|
229
|
+
```typescript
|
|
230
|
+
import { RateLimitError, ConcurrencyError } from 'gh-as-db';
|
|
231
|
+
|
|
232
|
+
try {
|
|
233
|
+
await users.create({ id: '1', name: 'Alice' });
|
|
234
|
+
} catch (error) {
|
|
235
|
+
if (error instanceof RateLimitError) {
|
|
236
|
+
console.log('Rate limited, retry after:', error.retryAfter);
|
|
237
|
+
}
|
|
238
|
+
if (error instanceof ConcurrencyError) {
|
|
239
|
+
console.log('Conflict, re-read and retry');
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
```
|
|
243
|
+
|
|
196
244
|
## CLI Usage
|
|
197
245
|
|
|
198
246
|
`gh-as-db` comes with a CLI tool to help you manage your repository.
|
|
@@ -217,10 +265,17 @@ For small projects, side-projects, or internal tools, setting up a database serv
|
|
|
217
265
|
|
|
218
266
|
## Performance
|
|
219
267
|
|
|
220
|
-
- **Consistent Caching**:
|
|
268
|
+
- **Consistent Caching**: With the default `cacheTTL: 0`, every read validates against GitHub using **Conditional GET** (`If-None-Match`) to ensure data is always up-to-date even across multiple instances (e.g., serverless), while minimizing API costs. Set `cacheTTL > 0` for additional in-memory caching with a possible staleness window.
|
|
221
269
|
- **Write-Through**: Updates the local cache immediately after a write, preventing 404s during redirects.
|
|
222
270
|
- **Indexing**: Automatic in-memory indexing on all fields makes querying fast even as data grows.
|
|
223
271
|
- **Optimistic Concurrency**: Uses Git SHAs to ensure that you don't overwrite changes made by another client.
|
|
272
|
+
- **Automatic Retries**: Transient failures and rate limits are handled transparently with exponential backoff.
|
|
273
|
+
|
|
274
|
+
## Security
|
|
275
|
+
|
|
276
|
+
- **Token handling**: The GitHub PAT is stored on `db.config.accessToken`. Avoid logging, serializing, or exposing the `db` object. The library's `toJSON()` method redacts the token if you need to serialize the config for debugging.
|
|
277
|
+
- **Path safety**: Collection names and item IDs are validated against path traversal attacks. Only alphanumeric characters, dots, hyphens, and underscores are permitted.
|
|
278
|
+
- **Input validation**: Query operators are validated and unknown operators throw errors. Pagination parameters are range-checked.
|
|
224
279
|
|
|
225
280
|
## License
|
|
226
281
|
|
package/dist/core/types.d.ts
CHANGED
|
@@ -1,9 +1,15 @@
|
|
|
1
|
+
export interface RetryConfig {
|
|
2
|
+
maxRetries?: number;
|
|
3
|
+
baseDelay?: number;
|
|
4
|
+
maxDelay?: number;
|
|
5
|
+
}
|
|
1
6
|
export interface GitHubDBConfig {
|
|
2
7
|
accessToken: string;
|
|
3
8
|
owner: string;
|
|
4
9
|
repo: string;
|
|
5
10
|
branch?: string;
|
|
6
11
|
cacheTTL?: number;
|
|
12
|
+
retry?: RetryConfig | false;
|
|
7
13
|
}
|
|
8
14
|
export type Schema = Record<string, any>;
|
|
9
15
|
export type MiddlewareOperation = "create" | "update" | "read" | "delete";
|
|
@@ -39,9 +45,18 @@ export declare class ConcurrencyError extends Error {
|
|
|
39
45
|
readonly path: string;
|
|
40
46
|
constructor(path: string);
|
|
41
47
|
}
|
|
48
|
+
export declare class RateLimitError extends Error {
|
|
49
|
+
readonly retryAfter?: number | undefined;
|
|
50
|
+
constructor(retryAfter?: number | undefined);
|
|
51
|
+
}
|
|
52
|
+
export declare class CorruptedContentError extends Error {
|
|
53
|
+
readonly path: string;
|
|
54
|
+
constructor(path: string, cause: Error);
|
|
55
|
+
}
|
|
42
56
|
export interface StorageResponse<T> {
|
|
43
57
|
data: T;
|
|
44
58
|
sha: string;
|
|
59
|
+
etag?: string;
|
|
45
60
|
}
|
|
46
61
|
export interface Validator<T> {
|
|
47
62
|
validate: (data: unknown) => Promise<T> | T;
|
|
@@ -60,7 +75,7 @@ export interface IStorageProvider {
|
|
|
60
75
|
listDirectory(path: string): Promise<{
|
|
61
76
|
path: string;
|
|
62
77
|
sha: string;
|
|
63
|
-
type: "file" | "dir";
|
|
78
|
+
type: "file" | "dir" | "symlink" | "submodule";
|
|
64
79
|
}[]>;
|
|
65
80
|
commit(changes: CommitChange[], message: string): Promise<string>;
|
|
66
81
|
}
|
package/dist/core/types.js
CHANGED
|
@@ -6,3 +6,20 @@ export class ConcurrencyError extends Error {
|
|
|
6
6
|
this.name = "ConcurrencyError";
|
|
7
7
|
}
|
|
8
8
|
}
|
|
9
|
+
export class RateLimitError extends Error {
|
|
10
|
+
retryAfter;
|
|
11
|
+
constructor(retryAfter) {
|
|
12
|
+
super(`GitHub API rate limit exceeded${retryAfter ? `. Retry after ${retryAfter}s` : ""}`);
|
|
13
|
+
this.retryAfter = retryAfter;
|
|
14
|
+
this.name = "RateLimitError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export class CorruptedContentError extends Error {
|
|
18
|
+
path;
|
|
19
|
+
constructor(path, cause) {
|
|
20
|
+
super(`Corrupted or malformed content at ${path}: ${cause.message}`);
|
|
21
|
+
this.path = path;
|
|
22
|
+
this.name = "CorruptedContentError";
|
|
23
|
+
this.cause = cause;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
const VALID_NAME_RE = /^[A-Za-z0-9._-]+$/;
|
|
2
|
+
export function validateName(name, context = "name") {
|
|
3
|
+
if (!name || typeof name !== "string") {
|
|
4
|
+
throw new Error(`Invalid ${context}: must be a non-empty string`);
|
|
5
|
+
}
|
|
6
|
+
if (name === "." || name === "..") {
|
|
7
|
+
throw new Error(`Invalid ${context}: "${name}" is a reserved name`);
|
|
8
|
+
}
|
|
9
|
+
if (name.includes("/") || name.includes("\\") || name.includes("\0")) {
|
|
10
|
+
throw new Error(`Invalid ${context}: "${name}" contains path separators or control characters`);
|
|
11
|
+
}
|
|
12
|
+
if (!VALID_NAME_RE.test(name)) {
|
|
13
|
+
throw new Error(`Invalid ${context}: "${name}" contains invalid characters`);
|
|
14
|
+
}
|
|
15
|
+
if (name.length > 255) {
|
|
16
|
+
throw new Error(`Invalid ${context}: exceeds maximum length of 255 characters`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function validateId(id) {
|
|
20
|
+
validateName(id, "id");
|
|
21
|
+
}
|
|
22
|
+
export function validatePath(path, context = "path") {
|
|
23
|
+
if (!path || typeof path !== "string") {
|
|
24
|
+
throw new Error(`Invalid ${context}: must be a non-empty string`);
|
|
25
|
+
}
|
|
26
|
+
if (path.startsWith("/")) {
|
|
27
|
+
throw new Error(`Invalid ${context}: "${path}" must not start with /`);
|
|
28
|
+
}
|
|
29
|
+
if (path.includes("..")) {
|
|
30
|
+
throw new Error(`Invalid ${context}: "${path}" contains path traversal ("..")`);
|
|
31
|
+
}
|
|
32
|
+
if (path.includes("\0")) {
|
|
33
|
+
throw new Error(`Invalid ${context}: "${path}" contains null byte`);
|
|
34
|
+
}
|
|
35
|
+
}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { GitHubDBConfig, IStorageProvider, StorageResponse, CommitChange } from "../core/types.js";
|
|
1
|
+
import { RetryConfig, GitHubDBConfig, IStorageProvider, StorageResponse, CommitChange } from "../core/types.js";
|
|
2
2
|
import { ICacheProvider } from "./cache-provider.js";
|
|
3
3
|
export declare class GitHubStorageProvider implements IStorageProvider {
|
|
4
4
|
private config;
|
|
@@ -6,8 +6,18 @@ export declare class GitHubStorageProvider implements IStorageProvider {
|
|
|
6
6
|
private cache;
|
|
7
7
|
private staleCache;
|
|
8
8
|
private readonly DEFAULT_TTL;
|
|
9
|
+
private readonly MAX_STALE_ENTRIES;
|
|
9
10
|
constructor(config: GitHubDBConfig, cache?: ICacheProvider);
|
|
11
|
+
private setStaleCache;
|
|
12
|
+
private parseRetryAfter;
|
|
13
|
+
private isSecondaryRateLimit;
|
|
14
|
+
private retryWithBackoff;
|
|
10
15
|
testConnection(): Promise<boolean>;
|
|
16
|
+
testConnectionDetailed(): Promise<{
|
|
17
|
+
ok: boolean;
|
|
18
|
+
reason?: "auth" | "not-found" | "rate-limit" | "network" | "unknown";
|
|
19
|
+
error?: Error;
|
|
20
|
+
}>;
|
|
11
21
|
exists(path: string): Promise<boolean>;
|
|
12
22
|
readJson<T>(path: string): Promise<StorageResponse<T>>;
|
|
13
23
|
writeJson<T>(path: string, content: T, message: string, sha?: string): Promise<string>;
|
|
@@ -16,6 +26,16 @@ export declare class GitHubStorageProvider implements IStorageProvider {
|
|
|
16
26
|
listDirectory(path: string): Promise<{
|
|
17
27
|
path: string;
|
|
18
28
|
sha: string;
|
|
19
|
-
type: "file" | "dir";
|
|
29
|
+
type: "file" | "dir" | "symlink" | "submodule";
|
|
20
30
|
}[]>;
|
|
31
|
+
toJSON(): {
|
|
32
|
+
config: {
|
|
33
|
+
accessToken: string;
|
|
34
|
+
owner: string;
|
|
35
|
+
repo: string;
|
|
36
|
+
branch?: string;
|
|
37
|
+
cacheTTL?: number;
|
|
38
|
+
retry?: RetryConfig | false;
|
|
39
|
+
};
|
|
40
|
+
};
|
|
21
41
|
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { Octokit } from "@octokit/rest";
|
|
2
|
-
import { ConcurrencyError, } from "../core/types.js";
|
|
2
|
+
import { ConcurrencyError, CorruptedContentError, RateLimitError, } from "../core/types.js";
|
|
3
3
|
import { MemoryCacheProvider } from "./cache-provider.js";
|
|
4
|
+
import { validatePath } from "../core/validation.js";
|
|
4
5
|
export class GitHubStorageProvider {
|
|
5
6
|
config;
|
|
6
7
|
octokit;
|
|
7
8
|
cache;
|
|
8
9
|
staleCache = new Map();
|
|
9
10
|
DEFAULT_TTL = 0; // Default to 0 for consistency
|
|
11
|
+
MAX_STALE_ENTRIES = 1000;
|
|
10
12
|
constructor(config, cache) {
|
|
11
13
|
this.config = config;
|
|
12
14
|
this.octokit = new Octokit({
|
|
@@ -14,28 +16,148 @@ export class GitHubStorageProvider {
|
|
|
14
16
|
});
|
|
15
17
|
this.cache = cache || new MemoryCacheProvider();
|
|
16
18
|
}
|
|
19
|
+
setStaleCache(path, entry) {
|
|
20
|
+
if (this.staleCache.size >= this.MAX_STALE_ENTRIES) {
|
|
21
|
+
const oldest = this.staleCache.keys().next().value;
|
|
22
|
+
if (oldest)
|
|
23
|
+
this.staleCache.delete(oldest);
|
|
24
|
+
}
|
|
25
|
+
this.staleCache.set(path, entry);
|
|
26
|
+
}
|
|
27
|
+
parseRetryAfter(value) {
|
|
28
|
+
const num = Number(value);
|
|
29
|
+
if (Number.isFinite(num) && num > 0) {
|
|
30
|
+
return num * 1000;
|
|
31
|
+
}
|
|
32
|
+
const date = Date.parse(value);
|
|
33
|
+
if (Number.isFinite(date)) {
|
|
34
|
+
return Math.max(0, date - Date.now());
|
|
35
|
+
}
|
|
36
|
+
return NaN;
|
|
37
|
+
}
|
|
38
|
+
isSecondaryRateLimit(error) {
|
|
39
|
+
const headers = error?.response?.headers || {};
|
|
40
|
+
if (headers["retry-after"])
|
|
41
|
+
return true;
|
|
42
|
+
const message = error?.message || "";
|
|
43
|
+
if (message.includes("secondary rate limit") ||
|
|
44
|
+
message.includes("abuse detection") ||
|
|
45
|
+
message.includes("abuse mechanism")) {
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
async retryWithBackoff(fn) {
|
|
51
|
+
if (this.config.retry === false)
|
|
52
|
+
return fn();
|
|
53
|
+
const { maxRetries = 3, baseDelay = 1000, maxDelay = 10000, } = this.config.retry ?? {};
|
|
54
|
+
let lastError;
|
|
55
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
56
|
+
try {
|
|
57
|
+
return await fn();
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
lastError = error;
|
|
61
|
+
const status = error?.status;
|
|
62
|
+
// Non-retryable: 409 (concurrency), 4xx client errors (except 429)
|
|
63
|
+
if (status === 409)
|
|
64
|
+
throw error;
|
|
65
|
+
if (status && status < 500 && status !== 429) {
|
|
66
|
+
if (status === 403 && this.isSecondaryRateLimit(error)) {
|
|
67
|
+
// Fall through to retry logic below
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// Retry status-less errors that appear to be network errors
|
|
74
|
+
if (!status) {
|
|
75
|
+
const code = error?.code;
|
|
76
|
+
const isNetworkError = code === "ENOTFOUND" ||
|
|
77
|
+
code === "ECONNRESET" ||
|
|
78
|
+
code === "ETIMEDOUT" ||
|
|
79
|
+
code === "EAI_AGAIN" ||
|
|
80
|
+
code === "ECONNREFUSED" ||
|
|
81
|
+
code === "EPIPE" ||
|
|
82
|
+
!error?.response;
|
|
83
|
+
if (!isNetworkError) {
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
// Fall through to retry logic below
|
|
87
|
+
}
|
|
88
|
+
// Last attempt — don't wait, just break
|
|
89
|
+
if (attempt === maxRetries)
|
|
90
|
+
break;
|
|
91
|
+
// Calculate delay
|
|
92
|
+
let delay;
|
|
93
|
+
if (status === 429 && error.response?.headers?.["retry-after"]) {
|
|
94
|
+
const retryAfter = error.response.headers["retry-after"];
|
|
95
|
+
delay = this.parseRetryAfter(retryAfter);
|
|
96
|
+
if (!Number.isFinite(delay) || delay < 0) {
|
|
97
|
+
delay = baseDelay * Math.pow(2, attempt);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
delay = baseDelay * Math.pow(2, attempt);
|
|
102
|
+
// Add jitter to avoid thundering herd on simultaneous retries
|
|
103
|
+
delay = delay * (0.5 + Math.random() * 0.5);
|
|
104
|
+
}
|
|
105
|
+
// Cap all delays (including Retry-After) against maxDelay
|
|
106
|
+
delay = Math.min(delay, maxDelay);
|
|
107
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Exhausted retries on 429 → RateLimitError
|
|
111
|
+
if (lastError.status === 429) {
|
|
112
|
+
const retryAfter = lastError.response?.headers?.["retry-after"];
|
|
113
|
+
throw new RateLimitError(retryAfter ? parseInt(retryAfter, 10) : undefined);
|
|
114
|
+
}
|
|
115
|
+
throw lastError;
|
|
116
|
+
}
|
|
17
117
|
async testConnection() {
|
|
18
118
|
try {
|
|
19
|
-
await this.octokit.repos.get({
|
|
119
|
+
await this.retryWithBackoff(() => this.octokit.repos.get({
|
|
20
120
|
owner: this.config.owner,
|
|
21
121
|
repo: this.config.repo,
|
|
22
|
-
});
|
|
122
|
+
}));
|
|
23
123
|
return true;
|
|
24
124
|
}
|
|
25
125
|
catch (error) {
|
|
26
126
|
return false;
|
|
27
127
|
}
|
|
28
128
|
}
|
|
129
|
+
async testConnectionDetailed() {
|
|
130
|
+
try {
|
|
131
|
+
await this.retryWithBackoff(() => this.octokit.repos.get({
|
|
132
|
+
owner: this.config.owner,
|
|
133
|
+
repo: this.config.repo,
|
|
134
|
+
}));
|
|
135
|
+
return { ok: true };
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
const status = error?.status;
|
|
139
|
+
if (status === 401)
|
|
140
|
+
return { ok: false, reason: "auth", error };
|
|
141
|
+
if (status === 404)
|
|
142
|
+
return { ok: false, reason: "not-found", error };
|
|
143
|
+
if (status === 429)
|
|
144
|
+
return { ok: false, reason: "rate-limit", error };
|
|
145
|
+
if (!status || error?.code === "ENOTFOUND" || error?.code === "ECONNRESET" || error?.code === "ETIMEDOUT") {
|
|
146
|
+
return { ok: false, reason: "network", error };
|
|
147
|
+
}
|
|
148
|
+
return { ok: false, reason: "unknown", error };
|
|
149
|
+
}
|
|
150
|
+
}
|
|
29
151
|
async exists(path) {
|
|
30
|
-
if (this.cache.get(path)
|
|
152
|
+
if (this.cache.get(path)) {
|
|
31
153
|
return true;
|
|
32
154
|
}
|
|
33
155
|
try {
|
|
34
|
-
await this.octokit.repos.getContent({
|
|
156
|
+
await this.retryWithBackoff(() => this.octokit.repos.getContent({
|
|
35
157
|
owner: this.config.owner,
|
|
36
158
|
repo: this.config.repo,
|
|
37
159
|
path,
|
|
38
|
-
});
|
|
160
|
+
}));
|
|
39
161
|
return true;
|
|
40
162
|
}
|
|
41
163
|
catch (error) {
|
|
@@ -54,33 +176,52 @@ export class GitHubStorageProvider {
|
|
|
54
176
|
// 2. Check stale cache for conditional request
|
|
55
177
|
const stale = this.staleCache.get(path);
|
|
56
178
|
try {
|
|
57
|
-
const response = await this.octokit.repos.getContent({
|
|
179
|
+
const response = await this.retryWithBackoff(() => this.octokit.repos.getContent({
|
|
58
180
|
owner: this.config.owner,
|
|
59
181
|
repo: this.config.repo,
|
|
60
182
|
path,
|
|
61
|
-
headers: stale
|
|
62
|
-
|
|
183
|
+
headers: stale
|
|
184
|
+
? {
|
|
185
|
+
"if-none-match": stale.etag || `"${stale.sha}"`,
|
|
186
|
+
}
|
|
187
|
+
: {},
|
|
188
|
+
}));
|
|
63
189
|
if (Array.isArray(response.data)) {
|
|
64
190
|
throw new Error("Path is a directory, not a file");
|
|
65
191
|
}
|
|
66
192
|
if (!("content" in response.data) || !("sha" in response.data)) {
|
|
67
193
|
throw new Error("No content or SHA in response");
|
|
68
194
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
bytes
|
|
195
|
+
let content;
|
|
196
|
+
try {
|
|
197
|
+
const binary = atob(response.data.content.replace(/\s/g, ""));
|
|
198
|
+
const bytes = new Uint8Array(binary.length);
|
|
199
|
+
for (let i = 0; i < binary.length; i++) {
|
|
200
|
+
bytes[i] = binary.charCodeAt(i);
|
|
201
|
+
}
|
|
202
|
+
const decoder = new TextDecoder("utf-8");
|
|
203
|
+
content = decoder.decode(bytes);
|
|
204
|
+
}
|
|
205
|
+
catch (e) {
|
|
206
|
+
throw new CorruptedContentError(path, e);
|
|
207
|
+
}
|
|
208
|
+
let parsed;
|
|
209
|
+
try {
|
|
210
|
+
parsed = JSON.parse(content);
|
|
73
211
|
}
|
|
74
|
-
|
|
75
|
-
|
|
212
|
+
catch (e) {
|
|
213
|
+
throw new CorruptedContentError(path, e);
|
|
214
|
+
}
|
|
215
|
+
const etag = response.headers?.etag || undefined;
|
|
76
216
|
const result = {
|
|
77
|
-
data:
|
|
217
|
+
data: parsed,
|
|
78
218
|
sha: response.data.sha,
|
|
219
|
+
etag,
|
|
79
220
|
};
|
|
80
221
|
// Update both caches
|
|
81
222
|
const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
|
|
82
223
|
this.cache.set(path, result, ttl);
|
|
83
|
-
this.
|
|
224
|
+
this.setStaleCache(path, result);
|
|
84
225
|
return result;
|
|
85
226
|
}
|
|
86
227
|
catch (error) {
|
|
@@ -102,11 +243,11 @@ export class GitHubStorageProvider {
|
|
|
102
243
|
}
|
|
103
244
|
else {
|
|
104
245
|
try {
|
|
105
|
-
const existing = await this.octokit.repos.getContent({
|
|
246
|
+
const existing = await this.retryWithBackoff(() => this.octokit.repos.getContent({
|
|
106
247
|
owner: this.config.owner,
|
|
107
248
|
repo: this.config.repo,
|
|
108
249
|
path,
|
|
109
|
-
});
|
|
250
|
+
}));
|
|
110
251
|
if (!Array.isArray(existing.data) && "sha" in existing.data) {
|
|
111
252
|
internalSha = existing.data.sha;
|
|
112
253
|
}
|
|
@@ -118,8 +259,11 @@ export class GitHubStorageProvider {
|
|
|
118
259
|
}
|
|
119
260
|
}
|
|
120
261
|
}
|
|
262
|
+
// Note: internalSha is resolved before the retry wrapper. If a write fails
|
|
263
|
+
// with a 5xx after GitHub has already committed the file (e.g. network timeout),
|
|
264
|
+
// the retry will reuse the stale sha and may fail with a 409.
|
|
121
265
|
try {
|
|
122
|
-
const response = await this.octokit.repos.createOrUpdateFileContents({
|
|
266
|
+
const response = await this.retryWithBackoff(() => this.octokit.repos.createOrUpdateFileContents({
|
|
123
267
|
owner: this.config.owner,
|
|
124
268
|
repo: this.config.repo,
|
|
125
269
|
path,
|
|
@@ -128,12 +272,15 @@ export class GitHubStorageProvider {
|
|
|
128
272
|
.map((b) => String.fromCharCode(b))
|
|
129
273
|
.join("")),
|
|
130
274
|
sha: internalSha,
|
|
131
|
-
});
|
|
132
|
-
const newSha = response.data.content?.sha
|
|
275
|
+
}));
|
|
276
|
+
const newSha = response.data.content?.sha;
|
|
277
|
+
if (!newSha) {
|
|
278
|
+
throw new Error(`writeJson succeeded but returned no SHA for ${path}`);
|
|
279
|
+
}
|
|
133
280
|
const result = { data: content, sha: newSha };
|
|
134
281
|
const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
|
|
135
282
|
this.cache.set(path, result, ttl);
|
|
136
|
-
this.
|
|
283
|
+
this.setStaleCache(path, result);
|
|
137
284
|
return newSha;
|
|
138
285
|
}
|
|
139
286
|
catch (error) {
|
|
@@ -147,21 +294,24 @@ export class GitHubStorageProvider {
|
|
|
147
294
|
if (changes.length === 0) {
|
|
148
295
|
throw new Error("No changes to commit");
|
|
149
296
|
}
|
|
297
|
+
for (const change of changes) {
|
|
298
|
+
validatePath(change.path, "commit path");
|
|
299
|
+
}
|
|
150
300
|
const branch = this.config.branch || "main";
|
|
151
301
|
try {
|
|
152
302
|
// 1. Get the current head SHA
|
|
153
|
-
const { data: ref } = await this.octokit.git.getRef({
|
|
303
|
+
const { data: ref } = await this.retryWithBackoff(() => this.octokit.git.getRef({
|
|
154
304
|
owner: this.config.owner,
|
|
155
305
|
repo: this.config.repo,
|
|
156
306
|
ref: `heads/${branch}`,
|
|
157
|
-
});
|
|
307
|
+
}));
|
|
158
308
|
const latestCommitSha = ref.object.sha;
|
|
159
309
|
// 2. Get the tree SHA of the latest commit
|
|
160
|
-
const { data: latestCommit } = await this.octokit.git.getCommit({
|
|
310
|
+
const { data: latestCommit } = await this.retryWithBackoff(() => this.octokit.git.getCommit({
|
|
161
311
|
owner: this.config.owner,
|
|
162
312
|
repo: this.config.repo,
|
|
163
313
|
commit_sha: latestCommitSha,
|
|
164
|
-
});
|
|
314
|
+
}));
|
|
165
315
|
const baseTreeSha = latestCommit.tree.sha;
|
|
166
316
|
// 3. Create a new tree with multiple files
|
|
167
317
|
const treeEntries = changes.map((change) => {
|
|
@@ -180,27 +330,27 @@ export class GitHubStorageProvider {
|
|
|
180
330
|
content: JSON.stringify(change.content, null, 2),
|
|
181
331
|
};
|
|
182
332
|
});
|
|
183
|
-
const { data: newTree } = await this.octokit.git.createTree({
|
|
333
|
+
const { data: newTree } = await this.retryWithBackoff(() => this.octokit.git.createTree({
|
|
184
334
|
owner: this.config.owner,
|
|
185
335
|
repo: this.config.repo,
|
|
186
336
|
base_tree: baseTreeSha,
|
|
187
337
|
tree: treeEntries,
|
|
188
|
-
});
|
|
338
|
+
}));
|
|
189
339
|
// 4. Create a new commit
|
|
190
|
-
const { data: newCommit } = await this.octokit.git.createCommit({
|
|
340
|
+
const { data: newCommit } = await this.retryWithBackoff(() => this.octokit.git.createCommit({
|
|
191
341
|
owner: this.config.owner,
|
|
192
342
|
repo: this.config.repo,
|
|
193
343
|
message,
|
|
194
344
|
tree: newTree.sha,
|
|
195
345
|
parents: [latestCommitSha],
|
|
196
|
-
});
|
|
346
|
+
}));
|
|
197
347
|
// 5. Update the reference
|
|
198
|
-
await this.octokit.git.updateRef({
|
|
348
|
+
await this.retryWithBackoff(() => this.octokit.git.updateRef({
|
|
199
349
|
owner: this.config.owner,
|
|
200
350
|
repo: this.config.repo,
|
|
201
351
|
ref: `heads/${branch}`,
|
|
202
352
|
sha: newCommit.sha,
|
|
203
|
-
});
|
|
353
|
+
}));
|
|
204
354
|
// 6. Update cache for all involved files
|
|
205
355
|
const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
|
|
206
356
|
for (const entry of newTree.tree) {
|
|
@@ -209,37 +359,51 @@ export class GitHubStorageProvider {
|
|
|
209
359
|
if (change) {
|
|
210
360
|
const result = { data: change.content, sha: entry.sha };
|
|
211
361
|
this.cache.set(entry.path, result, ttl);
|
|
212
|
-
this.
|
|
362
|
+
this.setStaleCache(entry.path, result);
|
|
213
363
|
}
|
|
214
364
|
}
|
|
215
365
|
}
|
|
366
|
+
for (const change of changes) {
|
|
367
|
+
if (change.content === null) {
|
|
368
|
+
this.cache.delete(change.path);
|
|
369
|
+
this.staleCache.delete(change.path);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
216
372
|
return newCommit.sha;
|
|
217
373
|
}
|
|
218
374
|
catch (error) {
|
|
219
|
-
if (error.status === 409) {
|
|
375
|
+
if (error.status === 409 || error.status === 422) {
|
|
220
376
|
throw new ConcurrencyError("batch-commit");
|
|
221
377
|
}
|
|
222
378
|
throw error;
|
|
223
379
|
}
|
|
224
380
|
}
|
|
225
381
|
async deleteFile(path, message, sha) {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
382
|
+
try {
|
|
383
|
+
await this.retryWithBackoff(() => this.octokit.repos.deleteFile({
|
|
384
|
+
owner: this.config.owner,
|
|
385
|
+
repo: this.config.repo,
|
|
386
|
+
path,
|
|
387
|
+
message,
|
|
388
|
+
sha,
|
|
389
|
+
}));
|
|
390
|
+
}
|
|
391
|
+
catch (error) {
|
|
392
|
+
if (error.status === 409) {
|
|
393
|
+
throw new ConcurrencyError(path);
|
|
394
|
+
}
|
|
395
|
+
throw error;
|
|
396
|
+
}
|
|
233
397
|
this.cache.delete(path);
|
|
234
398
|
this.staleCache.delete(path);
|
|
235
399
|
}
|
|
236
400
|
async listDirectory(path) {
|
|
237
401
|
try {
|
|
238
|
-
const response = await this.octokit.repos.getContent({
|
|
402
|
+
const response = await this.retryWithBackoff(() => this.octokit.repos.getContent({
|
|
239
403
|
owner: this.config.owner,
|
|
240
404
|
repo: this.config.repo,
|
|
241
405
|
path,
|
|
242
|
-
});
|
|
406
|
+
}));
|
|
243
407
|
if (!Array.isArray(response.data)) {
|
|
244
408
|
throw new Error("Path is not a directory");
|
|
245
409
|
}
|
|
@@ -256,4 +420,12 @@ export class GitHubStorageProvider {
|
|
|
256
420
|
throw error;
|
|
257
421
|
}
|
|
258
422
|
}
|
|
423
|
+
toJSON() {
|
|
424
|
+
return {
|
|
425
|
+
config: {
|
|
426
|
+
...this.config,
|
|
427
|
+
accessToken: "[redacted]",
|
|
428
|
+
},
|
|
429
|
+
};
|
|
430
|
+
}
|
|
259
431
|
}
|
|
@@ -15,7 +15,7 @@ export declare class TransactionStorageProvider implements IStorageProvider {
|
|
|
15
15
|
listDirectory(path: string): Promise<{
|
|
16
16
|
path: string;
|
|
17
17
|
sha: string;
|
|
18
|
-
type: "file" | "dir";
|
|
18
|
+
type: "file" | "dir" | "symlink" | "submodule";
|
|
19
19
|
}[]>;
|
|
20
20
|
commit(changes: CommitChange[], message: string): Promise<string>;
|
|
21
21
|
/**
|
|
@@ -13,13 +13,18 @@ export class TransactionStorageProvider {
|
|
|
13
13
|
}
|
|
14
14
|
async exists(path) {
|
|
15
15
|
if (this.pendingChanges.has(path)) {
|
|
16
|
-
return
|
|
16
|
+
return this.pendingChanges.get(path) !== null;
|
|
17
17
|
}
|
|
18
18
|
return this.baseStorage.exists(path);
|
|
19
19
|
}
|
|
20
20
|
async readJson(path) {
|
|
21
21
|
const pending = this.pendingChanges.get(path);
|
|
22
22
|
if (pending !== undefined) {
|
|
23
|
+
if (pending === null) {
|
|
24
|
+
throw Object.assign(new Error(`File not found in transaction: ${path}`), {
|
|
25
|
+
status: 404,
|
|
26
|
+
});
|
|
27
|
+
}
|
|
23
28
|
return {
|
|
24
29
|
data: pending,
|
|
25
30
|
sha: "transaction-pending-sha",
|
|
@@ -35,9 +40,30 @@ export class TransactionStorageProvider {
|
|
|
35
40
|
this.pendingChanges.set(path, null);
|
|
36
41
|
}
|
|
37
42
|
async listDirectory(path) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
43
|
+
const baseList = await this.baseStorage.listDirectory(path);
|
|
44
|
+
const basePaths = new Set(baseList.map((e) => e.path));
|
|
45
|
+
const merged = [...baseList];
|
|
46
|
+
for (const [pendingPath, content] of this.pendingChanges) {
|
|
47
|
+
if (!pendingPath.startsWith(path))
|
|
48
|
+
continue;
|
|
49
|
+
if (content === null) {
|
|
50
|
+
// Deletion — remove from list
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (!basePaths.has(pendingPath)) {
|
|
54
|
+
// New file — add to list
|
|
55
|
+
merged.push({
|
|
56
|
+
path: pendingPath,
|
|
57
|
+
sha: "transaction-pending-sha",
|
|
58
|
+
type: "file",
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// Filter out deleted paths
|
|
63
|
+
return merged.filter((entry) => {
|
|
64
|
+
const pending = this.pendingChanges.get(entry.path);
|
|
65
|
+
return pending !== undefined ? pending !== null : true;
|
|
66
|
+
});
|
|
41
67
|
}
|
|
42
68
|
async commit(changes, message) {
|
|
43
69
|
return this.baseStorage.commit(changes, message);
|
package/dist/ui/cli/commands.js
CHANGED
|
@@ -2,6 +2,7 @@ import enquirer from "enquirer";
|
|
|
2
2
|
import chalk from "chalk";
|
|
3
3
|
import { Octokit } from "@octokit/rest";
|
|
4
4
|
import { GitHubStorageProvider } from "../../infrastructure/github-storage.js";
|
|
5
|
+
import { validateName } from "../../core/validation.js";
|
|
5
6
|
export async function initCommand() {
|
|
6
7
|
console.log(chalk.yellow("\n--- Database Initialization ---\n"));
|
|
7
8
|
try {
|
|
@@ -29,14 +30,21 @@ export async function initCommand() {
|
|
|
29
30
|
accessToken: answers.accessToken,
|
|
30
31
|
});
|
|
31
32
|
process.stdout.write(chalk.blue("Verifying connection... "));
|
|
32
|
-
const
|
|
33
|
-
if (
|
|
33
|
+
const result = await storage.testConnectionDetailed();
|
|
34
|
+
if (result.ok) {
|
|
34
35
|
console.log(chalk.green("Success!"));
|
|
35
36
|
console.log(chalk.dim("\nYou can now use these credentials in your application configuration."));
|
|
36
37
|
}
|
|
37
38
|
else {
|
|
38
|
-
console.log(chalk.red(
|
|
39
|
-
|
|
39
|
+
console.log(chalk.red(`Failed — ${result.reason || "unknown error"}`));
|
|
40
|
+
if (result.reason === "auth")
|
|
41
|
+
console.log(chalk.red("Check your token and permissions."));
|
|
42
|
+
else if (result.reason === "not-found")
|
|
43
|
+
console.log(chalk.red("Check that the owner and repo are correct."));
|
|
44
|
+
else if (result.reason === "rate-limit")
|
|
45
|
+
console.log(chalk.red("You've been rate limited. Try again later."));
|
|
46
|
+
else
|
|
47
|
+
console.log(chalk.red("Please check your token permissions and repository details."));
|
|
40
48
|
}
|
|
41
49
|
}
|
|
42
50
|
catch (error) {
|
|
@@ -52,6 +60,8 @@ export async function listCollectionsCommand() {
|
|
|
52
60
|
console.log(chalk.red("Error: Environment variables GH_DB_OWNER, GH_DB_REPO, and GH_DB_TOKEN must be set."));
|
|
53
61
|
return;
|
|
54
62
|
}
|
|
63
|
+
validateName(owner, "owner");
|
|
64
|
+
validateName(repo, "repo");
|
|
55
65
|
const octokit = new Octokit({ auth });
|
|
56
66
|
try {
|
|
57
67
|
const { data } = await octokit.repos.getContent({
|
|
@@ -60,9 +70,13 @@ export async function listCollectionsCommand() {
|
|
|
60
70
|
path: "",
|
|
61
71
|
});
|
|
62
72
|
if (Array.isArray(data)) {
|
|
63
|
-
const
|
|
64
|
-
.filter((item) => item.name.endsWith(".json"))
|
|
73
|
+
const jsonCollections = data
|
|
74
|
+
.filter((item) => item.type === "file" && item.name.endsWith(".json"))
|
|
65
75
|
.map((item) => item.name.replace(".json", ""));
|
|
76
|
+
const dirCollections = data
|
|
77
|
+
.filter((item) => item.type === "dir")
|
|
78
|
+
.map((item) => `${item.name}/ (sharded)`);
|
|
79
|
+
const collections = [...jsonCollections, ...dirCollections];
|
|
66
80
|
if (collections.length === 0) {
|
|
67
81
|
console.log(chalk.dim("No collections found in the repository."));
|
|
68
82
|
}
|
|
@@ -74,7 +88,7 @@ export async function listCollectionsCommand() {
|
|
|
74
88
|
}
|
|
75
89
|
}
|
|
76
90
|
catch (error) {
|
|
77
|
-
console.error(chalk.red(`Error fetching collections:
|
|
91
|
+
console.error(chalk.red(`Error fetching collections: GitHub API error`));
|
|
78
92
|
}
|
|
79
93
|
}
|
|
80
94
|
export async function inspectCollectionCommand(name) {
|
|
@@ -86,23 +100,37 @@ export async function inspectCollectionCommand(name) {
|
|
|
86
100
|
console.log(chalk.red("Error: Environment variables GH_DB_OWNER, GH_DB_REPO, and GH_DB_TOKEN must be set."));
|
|
87
101
|
return;
|
|
88
102
|
}
|
|
103
|
+
validateName(name, "collection name");
|
|
89
104
|
const storage = new GitHubStorageProvider({
|
|
90
105
|
owner,
|
|
91
106
|
repo,
|
|
92
107
|
accessToken: auth,
|
|
93
108
|
});
|
|
94
109
|
try {
|
|
95
|
-
const
|
|
96
|
-
|
|
110
|
+
const singlePath = `${name}.json`;
|
|
111
|
+
const shardedPath = `${name}`;
|
|
112
|
+
let isSingleFile = await storage.exists(singlePath);
|
|
113
|
+
let isSharded = !isSingleFile ? await storage.exists(shardedPath) : false;
|
|
114
|
+
if (!isSingleFile && !isSharded) {
|
|
97
115
|
console.log(chalk.red(`Collection "${name}" does not exist.`));
|
|
98
116
|
return;
|
|
99
117
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
118
|
+
if (isSingleFile) {
|
|
119
|
+
const response = await storage.readJson(singlePath);
|
|
120
|
+
console.log(JSON.stringify(response.data, null, 2));
|
|
121
|
+
console.log(chalk.dim(`\nTotal records: ${response.data.length}`));
|
|
122
|
+
console.log(chalk.dim(`Current SHA: ${response.sha}`));
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
const files = await storage.listDirectory(shardedPath);
|
|
126
|
+
const jsonFiles = files.filter((f) => f.type === "file" && f.path.endsWith(".json"));
|
|
127
|
+
console.log(chalk.dim(`\nSharded collection with ${jsonFiles.length} documents:`));
|
|
128
|
+
for (const file of jsonFiles) {
|
|
129
|
+
console.log(chalk.cyan(` • ${file.path}`));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
104
132
|
}
|
|
105
133
|
catch (error) {
|
|
106
|
-
console.error(chalk.red(`Error inspecting collection:
|
|
134
|
+
console.error(chalk.red(`Error inspecting collection: GitHub API error`));
|
|
107
135
|
}
|
|
108
136
|
}
|
package/dist/ui/cli/index.js
CHANGED
|
@@ -13,7 +13,7 @@ console.log(boxen(chalk.bold.blue("gh-as-db CLI"), {
|
|
|
13
13
|
program
|
|
14
14
|
.name("gh-as-db")
|
|
15
15
|
.description("CLI to manage your GitHub-based database")
|
|
16
|
-
.version("0.
|
|
16
|
+
.version("2.0.0");
|
|
17
17
|
program
|
|
18
18
|
.command("init")
|
|
19
19
|
.description("Initialize connection and verify repository")
|
package/dist/ui/collection.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ConcurrencyError, } from "../core/types.js";
|
|
2
2
|
import { Indexer } from "../core/indexer.js";
|
|
3
|
+
import { validateName, validateId } from "../core/validation.js";
|
|
3
4
|
export class Collection {
|
|
4
5
|
name;
|
|
5
6
|
storage;
|
|
@@ -14,6 +15,7 @@ export class Collection {
|
|
|
14
15
|
constructor(name, storage, middlewareOrOptions) {
|
|
15
16
|
this.name = name;
|
|
16
17
|
this.storage = storage;
|
|
18
|
+
validateName(name);
|
|
17
19
|
if (Array.isArray(middlewareOrOptions)) {
|
|
18
20
|
this.middleware = middlewareOrOptions;
|
|
19
21
|
this.strategy = "single-file";
|
|
@@ -28,6 +30,7 @@ export class Collection {
|
|
|
28
30
|
return this.strategy === "sharded" ? `${this.name}/` : `${this.name}.json`;
|
|
29
31
|
}
|
|
30
32
|
getItemPath(id) {
|
|
33
|
+
validateId(id);
|
|
31
34
|
return this.strategy === "sharded"
|
|
32
35
|
? `${this.name}/${id}.json`
|
|
33
36
|
: `${this.name}.json`;
|
|
@@ -49,18 +52,21 @@ export class Collection {
|
|
|
49
52
|
}
|
|
50
53
|
}
|
|
51
54
|
if (this.strategy === "sharded") {
|
|
55
|
+
validateId(finalItem.id);
|
|
52
56
|
const itemPath = this.getItemPath(finalItem.id);
|
|
53
57
|
const sha = await this.storage.writeJson(itemPath, finalItem, `Create item ${finalItem.id} in ${this.name}`, this.shas.get(itemPath));
|
|
54
58
|
this.shas.set(itemPath, sha);
|
|
55
59
|
if (this.dataLoaded) {
|
|
56
60
|
const index = this.items.findIndex((i) => i.id === finalItem.id);
|
|
57
61
|
if (index > -1) {
|
|
62
|
+
const oldItem = this.items[index];
|
|
58
63
|
this.items[index] = finalItem;
|
|
64
|
+
this.indexer.update(oldItem, finalItem);
|
|
59
65
|
}
|
|
60
66
|
else {
|
|
61
67
|
this.items.push(finalItem);
|
|
68
|
+
this.indexer.add(finalItem);
|
|
62
69
|
}
|
|
63
|
-
this.indexer.add(finalItem);
|
|
64
70
|
}
|
|
65
71
|
return finalItem;
|
|
66
72
|
}
|
|
@@ -77,7 +83,8 @@ export class Collection {
|
|
|
77
83
|
}
|
|
78
84
|
}
|
|
79
85
|
catch (error) {
|
|
80
|
-
|
|
86
|
+
if (error?.status !== 404)
|
|
87
|
+
throw error;
|
|
81
88
|
}
|
|
82
89
|
}
|
|
83
90
|
items.push(finalItem);
|
|
@@ -115,19 +122,38 @@ export class Collection {
|
|
|
115
122
|
queryOrPredicate.filters[0].operator === "eq" &&
|
|
116
123
|
this.indexer.hasIndex(queryOrPredicate.filters[0].field)) {
|
|
117
124
|
const filter = queryOrPredicate.filters[0];
|
|
118
|
-
const
|
|
119
|
-
if (
|
|
125
|
+
const rawResults = this.indexer.query(filter.field, filter.value);
|
|
126
|
+
if (rawResults !== null) {
|
|
127
|
+
let results = (await Promise.all(rawResults.map(async (item) => {
|
|
128
|
+
let current = item;
|
|
129
|
+
for (const mw of this.middleware) {
|
|
130
|
+
if (mw.afterRead)
|
|
131
|
+
current = await mw.afterRead(current, context);
|
|
132
|
+
}
|
|
133
|
+
return current;
|
|
134
|
+
})));
|
|
135
|
+
if (queryOrPredicate) {
|
|
136
|
+
results = this.applyQueryOptions(results, queryOrPredicate);
|
|
137
|
+
}
|
|
120
138
|
return results;
|
|
121
139
|
}
|
|
122
140
|
}
|
|
123
141
|
// Fallback to full scan of current in-memory items
|
|
142
|
+
let cachedItems = (await Promise.all(this.items.map(async (item) => {
|
|
143
|
+
let current = item;
|
|
144
|
+
for (const mw of this.middleware) {
|
|
145
|
+
if (mw.afterRead)
|
|
146
|
+
current = await mw.afterRead(current, context);
|
|
147
|
+
}
|
|
148
|
+
return current;
|
|
149
|
+
})));
|
|
124
150
|
if (typeof queryOrPredicate === "function") {
|
|
125
|
-
return
|
|
151
|
+
return cachedItems.filter(queryOrPredicate);
|
|
126
152
|
}
|
|
127
153
|
if (queryOrPredicate) {
|
|
128
|
-
return this.applyQueryOptions(
|
|
154
|
+
return this.applyQueryOptions(cachedItems, queryOrPredicate);
|
|
129
155
|
}
|
|
130
|
-
return
|
|
156
|
+
return cachedItems;
|
|
131
157
|
}
|
|
132
158
|
if (this.strategy === "sharded") {
|
|
133
159
|
const files = await this.storage.listDirectory(this.name);
|
|
@@ -144,6 +170,7 @@ export class Collection {
|
|
|
144
170
|
this.lastSha = response.sha;
|
|
145
171
|
items = response.data;
|
|
146
172
|
}
|
|
173
|
+
const rawItems = items;
|
|
147
174
|
items = await Promise.all(items.map(async (item) => {
|
|
148
175
|
let currentItem = item;
|
|
149
176
|
for (const mw of this.middleware) {
|
|
@@ -153,10 +180,10 @@ export class Collection {
|
|
|
153
180
|
}
|
|
154
181
|
return currentItem;
|
|
155
182
|
}));
|
|
156
|
-
// Build index and store in
|
|
183
|
+
// Build index and store RAW items (before afterRead) in memory
|
|
157
184
|
if (!this.dataLoaded) {
|
|
158
|
-
this.items =
|
|
159
|
-
this.indexer.build(
|
|
185
|
+
this.items = rawItems;
|
|
186
|
+
this.indexer.build(rawItems, rawItems.length > 0 ? Object.keys(rawItems[0]) : []);
|
|
160
187
|
this.dataLoaded = true;
|
|
161
188
|
}
|
|
162
189
|
if (typeof queryOrPredicate === "function") {
|
|
@@ -169,10 +196,20 @@ export class Collection {
|
|
|
169
196
|
}
|
|
170
197
|
applyQueryOptions(items, options) {
|
|
171
198
|
let result = [...items];
|
|
199
|
+
const PROTOTYPE_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
|
200
|
+
const VALID_OPERATORS = new Set([
|
|
201
|
+
"eq", "neq", "gt", "gte", "lt", "lte", "contains", "in",
|
|
202
|
+
]);
|
|
172
203
|
// Apply filters
|
|
173
204
|
if (options.filters) {
|
|
174
205
|
for (const filter of options.filters) {
|
|
175
206
|
result = result.filter((item) => {
|
|
207
|
+
if (PROTOTYPE_KEYS.has(filter.field)) {
|
|
208
|
+
return true;
|
|
209
|
+
}
|
|
210
|
+
if (!VALID_OPERATORS.has(filter.operator)) {
|
|
211
|
+
throw new Error(`Unknown filter operator: "${filter.operator}". Supported: ${[...VALID_OPERATORS].join(", ")}`);
|
|
212
|
+
}
|
|
176
213
|
const val = item[filter.field];
|
|
177
214
|
switch (filter.operator) {
|
|
178
215
|
case "eq":
|
|
@@ -192,10 +229,12 @@ export class Collection {
|
|
|
192
229
|
return val.includes(filter.value);
|
|
193
230
|
}
|
|
194
231
|
return (typeof val === "string" && val.includes(filter.value));
|
|
195
|
-
case "in":
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
232
|
+
case "in": {
|
|
233
|
+
if (!Array.isArray(filter.value)) {
|
|
234
|
+
throw new Error(`"in" operator requires an array value, got ${typeof filter.value}`);
|
|
235
|
+
}
|
|
236
|
+
return filter.value.includes(val);
|
|
237
|
+
}
|
|
199
238
|
}
|
|
200
239
|
});
|
|
201
240
|
}
|
|
@@ -217,14 +256,33 @@ export class Collection {
|
|
|
217
256
|
// Apply pagination
|
|
218
257
|
if (options.pagination) {
|
|
219
258
|
const { limit, offset = 0 } = options.pagination;
|
|
259
|
+
if (typeof offset !== "number" || offset < 0 || !Number.isFinite(offset)) {
|
|
260
|
+
throw new Error(`Invalid pagination offset: ${offset}`);
|
|
261
|
+
}
|
|
262
|
+
if (limit !== undefined && (typeof limit !== "number" || limit < 0 || !Number.isFinite(limit))) {
|
|
263
|
+
throw new Error(`Invalid pagination limit: ${limit}`);
|
|
264
|
+
}
|
|
220
265
|
result = result.slice(offset, limit ? offset + limit : undefined);
|
|
221
266
|
}
|
|
222
267
|
return result;
|
|
223
268
|
}
|
|
224
269
|
async findById(id) {
|
|
270
|
+
validateId(id);
|
|
225
271
|
if (this.dataLoaded && this.indexer.hasIndex("id")) {
|
|
226
272
|
const results = this.indexer.query("id", id);
|
|
227
|
-
|
|
273
|
+
if (results && results.length > 0) {
|
|
274
|
+
let item = results[0];
|
|
275
|
+
const context = {
|
|
276
|
+
collection: this.name,
|
|
277
|
+
operation: "read",
|
|
278
|
+
};
|
|
279
|
+
for (const mw of this.middleware) {
|
|
280
|
+
if (mw.afterRead)
|
|
281
|
+
item = await mw.afterRead(item, context);
|
|
282
|
+
}
|
|
283
|
+
return item;
|
|
284
|
+
}
|
|
285
|
+
return null;
|
|
228
286
|
}
|
|
229
287
|
if (this.strategy === "sharded") {
|
|
230
288
|
try {
|
|
@@ -255,13 +313,20 @@ export class Collection {
|
|
|
255
313
|
return items.find((item) => item.id === id) || null;
|
|
256
314
|
}
|
|
257
315
|
async update(id, updates) {
|
|
258
|
-
|
|
316
|
+
validateId(id);
|
|
317
|
+
const items = [...(await this.find())];
|
|
259
318
|
const index = items.findIndex((item) => item.id === id);
|
|
260
319
|
if (index === -1) {
|
|
261
320
|
throw new Error(`Item with id ${id} not found in ${this.name}`);
|
|
262
321
|
}
|
|
263
322
|
const originalItem = items[index];
|
|
264
|
-
|
|
323
|
+
const safeUpdates = {};
|
|
324
|
+
for (const key of Object.keys(updates)) {
|
|
325
|
+
if (key !== "__proto__" && key !== "constructor" && key !== "prototype") {
|
|
326
|
+
safeUpdates[key] = updates[key];
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
items[index] = { ...items[index], ...safeUpdates };
|
|
265
330
|
let finalItem = items[index];
|
|
266
331
|
// 1. Validation
|
|
267
332
|
if (this.validator) {
|
|
@@ -278,7 +343,11 @@ export class Collection {
|
|
|
278
343
|
}
|
|
279
344
|
}
|
|
280
345
|
items[index] = finalItem;
|
|
346
|
+
validateId(id);
|
|
281
347
|
if (this.strategy === "sharded") {
|
|
348
|
+
if (updates.id !== undefined && updates.id !== id) {
|
|
349
|
+
throw new Error(`Cannot change id of a sharded item. Delete and re-create instead.`);
|
|
350
|
+
}
|
|
282
351
|
const itemPath = this.getItemPath(id);
|
|
283
352
|
try {
|
|
284
353
|
const sha = await this.storage.writeJson(itemPath, finalItem, `Update item ${id} in ${this.name}`, this.shas.get(itemPath));
|
|
@@ -309,6 +378,7 @@ export class Collection {
|
|
|
309
378
|
return items[index];
|
|
310
379
|
}
|
|
311
380
|
async delete(id) {
|
|
381
|
+
validateId(id);
|
|
312
382
|
const items = await this.find();
|
|
313
383
|
const index = items.findIndex((item) => item.id === id);
|
|
314
384
|
if (index === -1) {
|
|
@@ -316,22 +386,40 @@ export class Collection {
|
|
|
316
386
|
}
|
|
317
387
|
const itemToDelete = items[index];
|
|
318
388
|
const filtered = items.filter((_, i) => i !== index);
|
|
389
|
+
validateId(id);
|
|
319
390
|
if (this.strategy === "sharded") {
|
|
320
391
|
const itemPath = this.getItemPath(id);
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
392
|
+
try {
|
|
393
|
+
const sha = this.shas.get(itemPath);
|
|
394
|
+
if (!sha) {
|
|
395
|
+
const response = await this.storage.readJson(itemPath);
|
|
396
|
+
await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, response.sha);
|
|
397
|
+
}
|
|
398
|
+
else {
|
|
399
|
+
await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, sha);
|
|
400
|
+
}
|
|
401
|
+
this.shas.delete(itemPath);
|
|
326
402
|
}
|
|
327
|
-
|
|
328
|
-
|
|
403
|
+
catch (error) {
|
|
404
|
+
if (error instanceof ConcurrencyError) {
|
|
405
|
+
this.dataLoaded = false;
|
|
406
|
+
this.shas.delete(itemPath);
|
|
407
|
+
}
|
|
408
|
+
throw error;
|
|
329
409
|
}
|
|
330
|
-
this.shas.delete(itemPath);
|
|
331
410
|
}
|
|
332
411
|
else {
|
|
333
412
|
try {
|
|
334
|
-
|
|
413
|
+
if (filtered.length === 0) {
|
|
414
|
+
const fileExists = await this.storage.exists(this.path);
|
|
415
|
+
if (fileExists) {
|
|
416
|
+
const existing = await this.storage.readJson(this.path);
|
|
417
|
+
await this.storage.deleteFile(this.path, `Delete item ${id} from ${this.name}`, existing.sha);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
else {
|
|
421
|
+
this.lastSha = await this.storage.writeJson(this.path, filtered, `Delete item ${id} from ${this.name}`, this.lastSha);
|
|
422
|
+
}
|
|
335
423
|
}
|
|
336
424
|
catch (error) {
|
|
337
425
|
if (error instanceof ConcurrencyError) {
|
package/dist/ui/github-db.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { GitHubDBConfig, IStorageProvider, Middleware, Schema, Validator } from "../core/types.js";
|
|
1
|
+
import { GitHubDBConfig, IStorageProvider, Middleware, Schema, StorageStrategy, Validator } from "../core/types.js";
|
|
2
2
|
import { Collection } from "./collection.js";
|
|
3
3
|
import { Transaction } from "./transaction.js";
|
|
4
4
|
export declare class GitHubDB {
|
|
@@ -9,6 +9,14 @@ export declare class GitHubDB {
|
|
|
9
9
|
collection<T extends Schema>(name: string, options?: {
|
|
10
10
|
middleware?: Middleware<T>[];
|
|
11
11
|
validator?: Validator<T>;
|
|
12
|
+
strategy?: StorageStrategy;
|
|
12
13
|
}): Collection<T>;
|
|
13
|
-
transaction(fn: (tx: Transaction) => Promise<void>, message?: string): Promise<string>;
|
|
14
|
+
transaction(fn: (tx: Transaction) => Promise<void>, message?: string): Promise<string | null>;
|
|
15
|
+
toJSON(): {
|
|
16
|
+
config: {
|
|
17
|
+
owner: string;
|
|
18
|
+
repo: string;
|
|
19
|
+
accessToken: string;
|
|
20
|
+
};
|
|
21
|
+
};
|
|
14
22
|
}
|
package/dist/ui/github-db.js
CHANGED
package/dist/ui/transaction.d.ts
CHANGED
package/dist/ui/transaction.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gh-as-db",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Use a private GitHub repository as a database for your application.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -18,14 +18,9 @@
|
|
|
18
18
|
"files": [
|
|
19
19
|
"dist",
|
|
20
20
|
"README.md",
|
|
21
|
-
"LICENSE"
|
|
21
|
+
"LICENSE",
|
|
22
|
+
"CHANGELOG.md"
|
|
22
23
|
],
|
|
23
|
-
"scripts": {
|
|
24
|
-
"test": "vitest",
|
|
25
|
-
"test:run": "vitest run",
|
|
26
|
-
"build": "tsc",
|
|
27
|
-
"prepublishOnly": "npm run build"
|
|
28
|
-
},
|
|
29
24
|
"keywords": [
|
|
30
25
|
"github",
|
|
31
26
|
"database",
|
|
@@ -60,5 +55,10 @@
|
|
|
60
55
|
"chalk": "^5.4.1",
|
|
61
56
|
"commander": "^13.1.0",
|
|
62
57
|
"enquirer": "^2.4.1"
|
|
58
|
+
},
|
|
59
|
+
"scripts": {
|
|
60
|
+
"test": "vitest",
|
|
61
|
+
"test:run": "vitest run",
|
|
62
|
+
"build": "tsc"
|
|
63
63
|
}
|
|
64
|
-
}
|
|
64
|
+
}
|