@uploadista/data-store-gcs 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/.turbo/turbo-check.log +5 -0
- package/LICENSE +21 -0
- package/README.md +479 -0
- package/dist/examples.d.ts +44 -0
- package/dist/examples.d.ts.map +1 -0
- package/dist/examples.js +82 -0
- package/dist/gcs-store-rest.d.ts +16 -0
- package/dist/gcs-store-rest.d.ts.map +1 -0
- package/dist/gcs-store-rest.js +188 -0
- package/dist/gcs-store-v2.d.ts +13 -0
- package/dist/gcs-store-v2.d.ts.map +1 -0
- package/dist/gcs-store-v2.js +190 -0
- package/dist/gcs-store.d.ts +12 -0
- package/dist/gcs-store.d.ts.map +1 -0
- package/dist/gcs-store.js +282 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/services/gcs-client-nodejs.service.d.ts +4 -0
- package/dist/services/gcs-client-nodejs.service.d.ts.map +1 -0
- package/dist/services/gcs-client-nodejs.service.js +312 -0
- package/dist/services/gcs-client-rest.service.d.ts +4 -0
- package/dist/services/gcs-client-rest.service.d.ts.map +1 -0
- package/dist/services/gcs-client-rest.service.js +299 -0
- package/dist/services/gcs-client.service.d.ts +56 -0
- package/dist/services/gcs-client.service.d.ts.map +1 -0
- package/dist/services/gcs-client.service.js +3 -0
- package/dist/services/index.d.ts +4 -0
- package/dist/services/index.d.ts.map +1 -0
- package/dist/services/index.js +3 -0
- package/package.json +31 -0
- package/src/gcs-store-v2.ts +286 -0
- package/src/gcs-store.ts +398 -0
- package/src/index.ts +6 -0
- package/src/services/gcs-client-nodejs.service.ts +435 -0
- package/src/services/gcs-client-rest.service.ts +406 -0
- package/src/services/gcs-client.service.ts +117 -0
- package/src/services/index.ts +3 -0
- package/tsconfig.json +12 -0
- package/tsconfig.tsbuildinfo +1 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 uploadista
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
# @uploadista/data-store-gcs
|
|
2
|
+
|
|
3
|
+
Google Cloud Storage data store for Uploadista - Store files in Google Cloud.
|
|
4
|
+
|
|
5
|
+
Provides GCS-based file storage with resumable uploads, cross-platform support (Node.js and browsers), and comprehensive error handling. Supports both legacy Node.js SDK and new REST-based implementation for edge environments.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- **Cross-Platform** - Works in Node.js, browsers, and edge environments
|
|
10
|
+
- **Dual Implementations** - Optimized Node.js and REST-based APIs
|
|
11
|
+
- **Resumable Uploads** - Automatic resume on connection failures
|
|
12
|
+
- **Streaming Support** - Efficient memory usage with stream-based uploads
|
|
13
|
+
- **Full Observability** - Metrics, logging, and distributed tracing
|
|
14
|
+
- **Metadata Support** - Attach custom metadata to objects
|
|
15
|
+
- **Automatic Retry** - Intelligent retry logic with exponential backoff
|
|
16
|
+
- **TypeScript** - Full type safety with comprehensive JSDoc
|
|
17
|
+
|
|
18
|
+
## Installation
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
npm install @uploadista/data-store-gcs @google-cloud/storage @uploadista/core
|
|
22
|
+
# or
|
|
23
|
+
pnpm add @uploadista/data-store-gcs @google-cloud/storage @uploadista/core
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Requirements
|
|
27
|
+
|
|
28
|
+
- Node.js 18+ (for Node.js implementation)
|
|
29
|
+
- Google Cloud project with Cloud Storage bucket
|
|
30
|
+
- Service account JSON key or Application Default Credentials
|
|
31
|
+
- TypeScript 5.0+ (optional but recommended)
|
|
32
|
+
|
|
33
|
+
## Quick Start
|
|
34
|
+
|
|
35
|
+
### 1. Create GCS Data Store (Node.js)
|
|
36
|
+
|
|
37
|
+
```typescript
|
|
38
|
+
import { gcsStoreNodejs } from "@uploadista/data-store-gcs";
|
|
39
|
+
import { createUploadServerLayer } from "@uploadista/server";
|
|
40
|
+
import { memoryKvStore } from "@uploadista/kv-store-memory";
|
|
41
|
+
import { webSocketEventEmitter } from "@uploadista/event-emitter-websocket";
|
|
42
|
+
|
|
43
|
+
// Create GCS store (Node.js optimized)
|
|
44
|
+
const gcsStore = gcsStoreNodejs({
|
|
45
|
+
bucketName: "my-uploads",
|
|
46
|
+
kvStore: memoryKvStore,
|
|
47
|
+
keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
// Use in upload server
|
|
51
|
+
const uploadLayer = createUploadServerLayer({
|
|
52
|
+
dataStore: gcsStore,
|
|
53
|
+
kvStore: memoryKvStore,
|
|
54
|
+
eventEmitter: webSocketEventEmitter,
|
|
55
|
+
});
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### 2. Create GCS Data Store (Cross-Platform REST)
|
|
59
|
+
|
|
60
|
+
```typescript
|
|
61
|
+
import { gcsStoreRest } from "@uploadista/data-store-gcs";
|
|
62
|
+
|
|
63
|
+
// Create GCS store (works in browsers and edge)
|
|
64
|
+
const gcsStore = gcsStoreRest({
|
|
65
|
+
bucketName: "my-uploads",
|
|
66
|
+
kvStore: memoryKvStore,
|
|
67
|
+
credentials: JSON.parse(process.env.GCS_CREDENTIALS!),
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
// Works in browsers, Cloudflare Workers, etc.
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### 3. Configure Google Cloud Credentials
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# Option 1: Service account key file (Node.js)
|
|
77
|
+
export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account-key.json
|
|
78
|
+
|
|
79
|
+
# Option 2: Application Default Credentials
|
|
80
|
+
gcloud auth application-default login
|
|
81
|
+
|
|
82
|
+
# Option 3: Embed credentials in environment
|
|
83
|
+
export GCS_CREDENTIALS='{"type":"service_account","project_id":"...", ...}'
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Configuration
|
|
87
|
+
|
|
88
|
+
### `GCSStoreOptions`
|
|
89
|
+
|
|
90
|
+
```typescript
|
|
91
|
+
type GCSStoreOptions = {
|
|
92
|
+
// Required
|
|
93
|
+
bucketName: string; // GCS bucket name
|
|
94
|
+
kvStore: KvStore<UploadFile>; // Metadata store
|
|
95
|
+
|
|
96
|
+
// Authentication - choose one:
|
|
97
|
+
keyFilename?: string; // Path to service account JSON key
|
|
98
|
+
credentials?: object; // Service account object (for embed)
|
|
99
|
+
};
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Available Implementations
|
|
103
|
+
|
|
104
|
+
```typescript
|
|
105
|
+
import {
|
|
106
|
+
gcsStore, // Legacy Node.js (uses @google-cloud/storage)
|
|
107
|
+
gcsStoreNodejs, // New Node.js (service-based, recommended)
|
|
108
|
+
gcsStoreRest, // REST-based (cross-platform)
|
|
109
|
+
} from "@uploadista/data-store-gcs";
|
|
110
|
+
|
|
111
|
+
// Recommended: Use service-based implementations
|
|
112
|
+
const nodeStore = gcsStoreNodejs(options); // Node.js optimized
|
|
113
|
+
const restStore = gcsStoreRest(options); // Cross-platform
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Google Cloud Setup Guide
|
|
117
|
+
|
|
118
|
+
### 1. Create GCS Bucket
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
gsutil mb -l us-central1 gs://my-uploads-prod
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### 2. Create Service Account
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
# Create service account
|
|
128
|
+
gcloud iam service-accounts create uploadista-service \
|
|
129
|
+
--display-name "Uploadista Service"
|
|
130
|
+
|
|
131
|
+
# Grant Storage admin role
|
|
132
|
+
gcloud projects add-iam-policy-binding my-project \
|
|
133
|
+
--member serviceAccount:uploadista-service@my-project.iam.gserviceaccount.com \
|
|
134
|
+
--role roles/storage.admin
|
|
135
|
+
|
|
136
|
+
# Create and download key
|
|
137
|
+
gcloud iam service-accounts keys create key.json \
|
|
138
|
+
--iam-account uploadista-service@my-project.iam.gserviceaccount.com
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### 3. Configure Bucket Permissions
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
# Make bucket public (optional)
|
|
145
|
+
gsutil acl ch -u AllUsers:R gs://my-uploads-prod
|
|
146
|
+
|
|
147
|
+
# Or set via IAM (recommended)
|
|
148
|
+
gcloud storage buckets set-iam-policy gs://my-uploads-prod policy.json
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
policy.json:
|
|
152
|
+
```json
|
|
153
|
+
{
|
|
154
|
+
"bindings": [
|
|
155
|
+
{
|
|
156
|
+
"members": ["serviceAccount:uploadista-service@my-project.iam.gserviceaccount.com"],
|
|
157
|
+
"role": "roles/storage.objectCreator"
|
|
158
|
+
}
|
|
159
|
+
]
|
|
160
|
+
}
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### 4. Enable APIs
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
gcloud services enable storage-api
|
|
167
|
+
gcloud services enable storage-component.googleapis.com
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### 5. Configure CORS (Optional)
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
gsutil cors set cors.json gs://my-uploads-prod
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
cors.json:
|
|
177
|
+
```json
|
|
178
|
+
[
|
|
179
|
+
{
|
|
180
|
+
"origin": ["https://myapp.com"],
|
|
181
|
+
"method": ["GET", "PUT", "POST"],
|
|
182
|
+
"responseHeader": ["Content-Type"],
|
|
183
|
+
"maxAgeSeconds": 3600
|
|
184
|
+
}
|
|
185
|
+
]
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
## Complete Server Example
|
|
189
|
+
|
|
190
|
+
```typescript
|
|
191
|
+
import Fastify from "fastify";
|
|
192
|
+
import WebSocket from "@fastify/websocket";
|
|
193
|
+
import JwT from "@fastify/jwt";
|
|
194
|
+
import { createFastifyUploadistaAdapter } from "@uploadista/adapters-fastify";
|
|
195
|
+
import { gcsStoreNodejs } from "@uploadista/data-store-gcs";
|
|
196
|
+
import { redisKvStore } from "@uploadista/kv-store-redis";
|
|
197
|
+
import { webSocketEventEmitter } from "@uploadista/event-emitter-websocket";
|
|
198
|
+
import { memoryEventBroadcaster } from "@uploadista/event-broadcaster-memory";
|
|
199
|
+
|
|
200
|
+
const fastify = Fastify({ logger: true });
|
|
201
|
+
|
|
202
|
+
await fastify.register(JwT, { secret: process.env.JWT_SECRET! });
|
|
203
|
+
await fastify.register(WebSocket);
|
|
204
|
+
|
|
205
|
+
// Configure GCS (Node.js)
|
|
206
|
+
const gcsStore = gcsStoreNodejs({
|
|
207
|
+
bucketName: process.env.GCS_BUCKET!,
|
|
208
|
+
kvStore: redisKvStore,
|
|
209
|
+
keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS,
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
// Create adapter
|
|
213
|
+
const adapter = await createFastifyUploadistaAdapter({
|
|
214
|
+
baseUrl: "uploadista",
|
|
215
|
+
dataStore: gcsStore,
|
|
216
|
+
kvStore: redisKvStore,
|
|
217
|
+
eventEmitter: webSocketEventEmitter,
|
|
218
|
+
eventBroadcaster: memoryEventBroadcaster,
|
|
219
|
+
flows: createFlowsEffect,
|
|
220
|
+
authMiddleware: async (req, reply) => {
|
|
221
|
+
try {
|
|
222
|
+
await req.jwtVerify();
|
|
223
|
+
return {
|
|
224
|
+
clientId: (req.user as any).sub,
|
|
225
|
+
permissions: ["upload:create"],
|
|
226
|
+
};
|
|
227
|
+
} catch {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
},
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
// Routes
|
|
234
|
+
fastify.all(`/${adapter.baseUrl}/*`, (req, res) => adapter.handler(req, res));
|
|
235
|
+
fastify.get("/ws", { websocket: true }, (socket, req) => {
|
|
236
|
+
adapter.websocketHandler(socket, req);
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
await fastify.listen({ port: 3000 });
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
## Cross-Platform Example (Browsers)
|
|
243
|
+
|
|
244
|
+
```typescript
|
|
245
|
+
import { gcsStoreRest } from "@uploadista/data-store-gcs";
|
|
246
|
+
|
|
247
|
+
// In browser or edge environment
|
|
248
|
+
const gcsStore = gcsStoreRest({
|
|
249
|
+
bucketName: "my-uploads-prod",
|
|
250
|
+
kvStore: memoryKvStore,
|
|
251
|
+
credentials: {
|
|
252
|
+
type: "service_account",
|
|
253
|
+
project_id: "my-project",
|
|
254
|
+
private_key_id: "...",
|
|
255
|
+
private_key: "...",
|
|
256
|
+
client_email: "...",
|
|
257
|
+
client_id: "...",
|
|
258
|
+
auth_uri: "https://accounts.google.com/o/oauth2/auth",
|
|
259
|
+
token_uri: "https://oauth2.googleapis.com/token",
|
|
260
|
+
auth_provider_x509_cert_url: "https://www.googleapis.com/oauth2/v1/certs",
|
|
261
|
+
},
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
// Works in browsers, Cloudflare Workers, etc.
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
## Performance Tuning
|
|
268
|
+
|
|
269
|
+
### Node.js Optimization
|
|
270
|
+
|
|
271
|
+
```typescript
|
|
272
|
+
const gcsStore = gcsStoreNodejs({
|
|
273
|
+
bucketName: "my-uploads-prod",
|
|
274
|
+
kvStore: redisKvStore,
|
|
275
|
+
keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS,
|
|
276
|
+
// Node.js SDK handles parallelization automatically
|
|
277
|
+
});
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
### REST API Optimization
|
|
281
|
+
|
|
282
|
+
```typescript
|
|
283
|
+
const gcsStore = gcsStoreRest({
|
|
284
|
+
bucketName: "my-uploads-prod",
|
|
285
|
+
kvStore: redisKvStore,
|
|
286
|
+
credentials: require("./service-account-key.json"),
|
|
287
|
+
});
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
## Environment Configuration
|
|
291
|
+
|
|
292
|
+
### .env File
|
|
293
|
+
|
|
294
|
+
```env
|
|
295
|
+
# GCS Configuration
|
|
296
|
+
GCS_BUCKET=my-uploads-prod
|
|
297
|
+
GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account-key.json
|
|
298
|
+
|
|
299
|
+
# Or embed credentials
|
|
300
|
+
GCS_CREDENTIALS='{"type":"service_account","project_id":"my-project",...}'
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### Using Service Account Key Files
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
# Download from Google Cloud Console
|
|
307
|
+
gcloud iam service-accounts keys create key.json \
|
|
308
|
+
--iam-account=uploadista@my-project.iam.gserviceaccount.com
|
|
309
|
+
|
|
310
|
+
# Export as environment variable
|
|
311
|
+
export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/key.json"
|
|
312
|
+
|
|
313
|
+
# Or use directly
|
|
314
|
+
const gcsStore = gcsStoreNodejs({
|
|
315
|
+
bucketName: "my-uploads",
|
|
316
|
+
kvStore,
|
|
317
|
+
keyFilename: "./service-account-key.json",
|
|
318
|
+
});
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
## Migration from Legacy Implementation
|
|
322
|
+
|
|
323
|
+
The store provides backward compatibility:
|
|
324
|
+
|
|
325
|
+
```typescript
|
|
326
|
+
import {
|
|
327
|
+
gcsStore, // Legacy - still works
|
|
328
|
+
gcsStoreNodejs, // Recommended new Node.js
|
|
329
|
+
} from "@uploadista/data-store-gcs";
|
|
330
|
+
|
|
331
|
+
// Legacy code still works
|
|
332
|
+
const legacyStore = gcsStore(options);
|
|
333
|
+
|
|
334
|
+
// But use new implementation for better features
|
|
335
|
+
const modernStore = gcsStoreNodejs(options);
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
## Error Handling
|
|
339
|
+
|
|
340
|
+
Common GCS errors and solutions:
|
|
341
|
+
|
|
342
|
+
| Error | Cause | Solution |
|
|
343
|
+
|-------|-------|----------|
|
|
344
|
+
| PERMISSION_DENIED | Insufficient IAM permissions | Check service account roles |
|
|
345
|
+
| NOT_FOUND | Bucket doesn't exist | Verify bucket name and project |
|
|
346
|
+
| INVALID_ARGUMENT | Invalid bucket name | Use lowercase, 3-63 chars |
|
|
347
|
+
| DEADLINE_EXCEEDED | Upload timeout | Increase timeout, reduce file size |
|
|
348
|
+
| UNAUTHENTICATED | Missing credentials | Set GOOGLE_APPLICATION_CREDENTIALS |
|
|
349
|
+
|
|
350
|
+
## Monitoring & Observability
|
|
351
|
+
|
|
352
|
+
```typescript
|
|
353
|
+
// Metrics automatically tracked:
|
|
354
|
+
// - gcs.upload.started
|
|
355
|
+
// - gcs.upload.progress
|
|
356
|
+
// - gcs.upload.completed
|
|
357
|
+
// - gcs.upload.failed
|
|
358
|
+
// - gcs.metadata.operations
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
## Deployment Examples
|
|
362
|
+
|
|
363
|
+
### Docker
|
|
364
|
+
|
|
365
|
+
```dockerfile
|
|
366
|
+
FROM node:20-alpine
|
|
367
|
+
WORKDIR /app
|
|
368
|
+
COPY package*.json ./
|
|
369
|
+
RUN npm ci --only=production
|
|
370
|
+
COPY dist ./dist
|
|
371
|
+
COPY service-account-key.json ./key.json
|
|
372
|
+
|
|
373
|
+
ENV NODE_ENV=production
|
|
374
|
+
ENV GOOGLE_APPLICATION_CREDENTIALS=/app/key.json
|
|
375
|
+
|
|
376
|
+
EXPOSE 3000
|
|
377
|
+
CMD ["node", "dist/server.js"]
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
### Google Cloud Run
|
|
381
|
+
|
|
382
|
+
```bash
|
|
383
|
+
# Create service account
|
|
384
|
+
gcloud iam service-accounts create uploadista
|
|
385
|
+
|
|
386
|
+
# Grant permissions
|
|
387
|
+
gcloud projects add-iam-policy-binding PROJECT_ID \
|
|
388
|
+
--member serviceAccount:uploadista@PROJECT_ID.iam.gserviceaccount.com \
|
|
389
|
+
--role roles/storage.objectAdmin
|
|
390
|
+
|
|
391
|
+
# Deploy
|
|
392
|
+
gcloud run deploy uploadista-server \
|
|
393
|
+
--source . \
|
|
394
|
+
--platform managed \
|
|
395
|
+
--memory 2Gi \
|
|
396
|
+
--service-account uploadista@PROJECT_ID.iam.gserviceaccount.com \
|
|
397
|
+
--set-env-vars GCS_BUCKET=my-uploads-prod
|
|
398
|
+
```
|
|
399
|
+
|
|
400
|
+
### Google App Engine
|
|
401
|
+
|
|
402
|
+
app.yaml:
|
|
403
|
+
```yaml
|
|
404
|
+
runtime: nodejs20
|
|
405
|
+
env: flex
|
|
406
|
+
|
|
407
|
+
env_variables:
|
|
408
|
+
GCS_BUCKET: "my-uploads-prod"
|
|
409
|
+
|
|
410
|
+
automatic_scaling:
|
|
411
|
+
min_instances: 1
|
|
412
|
+
max_instances: 10
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
## Related Packages
|
|
416
|
+
|
|
417
|
+
- **[@uploadista/data-store-s3](../s3/)** - AWS S3 storage
|
|
418
|
+
- **[@uploadista/data-store-azure](../azure/)** - Azure Blob Storage
|
|
419
|
+
- **[@uploadista/data-store-filesystem](../filesystem/)** - Local filesystem
|
|
420
|
+
- **[@uploadista/server](../../servers/server/)** - Core server utilities
|
|
421
|
+
- **[@uploadista/kv-store-redis](../../kv-stores/redis/)** - Redis KV store
|
|
422
|
+
- **[@uploadista/core](../../core/)** - Core engine
|
|
423
|
+
|
|
424
|
+
## TypeScript Support
|
|
425
|
+
|
|
426
|
+
Full TypeScript support with comprehensive types:
|
|
427
|
+
|
|
428
|
+
```typescript
|
|
429
|
+
import type { GCSStoreOptions } from "@uploadista/data-store-gcs";
|
|
430
|
+
import { gcsStoreNodejs, gcsStoreRest } from "@uploadista/data-store-gcs";
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
## Troubleshooting
|
|
434
|
+
|
|
435
|
+
### Permission Denied
|
|
436
|
+
|
|
437
|
+
```bash
|
|
438
|
+
# Check service account permissions
|
|
439
|
+
gcloud projects get-iam-policy my-project \
|
|
440
|
+
--flatten="bindings[].members" \
|
|
441
|
+
--filter="bindings.members:serviceAccount:uploadista@*"
|
|
442
|
+
|
|
443
|
+
# Grant Storage Object Creator role
|
|
444
|
+
gcloud projects add-iam-policy-binding my-project \
|
|
445
|
+
--member serviceAccount:uploadista@my-project.iam.gserviceaccount.com \
|
|
446
|
+
--role roles/storage.objectCreator
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
### Bucket Not Found
|
|
450
|
+
|
|
451
|
+
```bash
|
|
452
|
+
# List buckets
|
|
453
|
+
gsutil ls
|
|
454
|
+
|
|
455
|
+
# Check bucket name spelling
|
|
456
|
+
gsutil ls gs://my-uploads-prod/
|
|
457
|
+
```
|
|
458
|
+
|
|
459
|
+
### Slow Uploads
|
|
460
|
+
|
|
461
|
+
- Use `gcsStoreNodejs` in Node.js (more optimized)
|
|
462
|
+
- Check network bandwidth to GCS
|
|
463
|
+
- Verify service account has appropriate permissions
|
|
464
|
+
- Monitor GCS quota usage
|
|
465
|
+
|
|
466
|
+
### Authentication Issues
|
|
467
|
+
|
|
468
|
+
```bash
|
|
469
|
+
# Test credentials
|
|
470
|
+
gcloud auth activate-service-account \
|
|
471
|
+
--key-file=service-account-key.json
|
|
472
|
+
|
|
473
|
+
# List buckets with service account
|
|
474
|
+
gsutil -i serviceAccount@project.iam.gserviceaccount.com ls
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
## License
|
|
478
|
+
|
|
479
|
+
MIT
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage examples for GCS store implementations
|
|
3
|
+
*/
|
|
4
|
+
export declare function createNodeJSGCSStore(config: {
|
|
5
|
+
bucketName: string;
|
|
6
|
+
keyFilename?: string;
|
|
7
|
+
credentials?: object;
|
|
8
|
+
projectId?: string;
|
|
9
|
+
}): any;
|
|
10
|
+
export declare function createCloudflareWorkerGCSStore(config: {
|
|
11
|
+
bucketName: string;
|
|
12
|
+
accessToken: string;
|
|
13
|
+
projectId?: string;
|
|
14
|
+
}): any;
|
|
15
|
+
export declare function createGCSStore(config: {
|
|
16
|
+
bucketName: string;
|
|
17
|
+
keyFilename?: string;
|
|
18
|
+
credentials?: object;
|
|
19
|
+
accessToken?: string;
|
|
20
|
+
projectId?: string;
|
|
21
|
+
}): any;
|
|
22
|
+
/**
|
|
23
|
+
* Usage in your application:
|
|
24
|
+
*
|
|
25
|
+
* // Node.js
|
|
26
|
+
* const nodeJSStore = createNodeJSGCSStore({
|
|
27
|
+
* bucketName: "my-bucket",
|
|
28
|
+
* keyFilename: "/path/to/service-account.json"
|
|
29
|
+
* });
|
|
30
|
+
*
|
|
31
|
+
* // Cloudflare Workers
|
|
32
|
+
* const workersStore = createCloudflareWorkerGCSStore({
|
|
33
|
+
* bucketName: "my-bucket",
|
|
34
|
+
* accessToken: "ya29.xxx"
|
|
35
|
+
* });
|
|
36
|
+
*
|
|
37
|
+
* // Environment-aware
|
|
38
|
+
* const store = createGCSStore({
|
|
39
|
+
* bucketName: "my-bucket",
|
|
40
|
+
* keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, // Node.js
|
|
41
|
+
* accessToken: process.env.GCS_ACCESS_TOKEN, // Workers
|
|
42
|
+
* });
|
|
43
|
+
*/
|
|
44
|
+
//# sourceMappingURL=examples.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"examples.d.ts","sourceRoot":"","sources":["../src/examples.ts"],"names":[],"mappings":"AAAA;;GAEG;AAYH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,OAiBA;AAGD,wBAAgB,8BAA8B,CAAC,MAAM,EAAE;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,OAgBA;AAGD,wBAAgB,cAAc,CAAC,MAAM,EAAE;IACrC,UAAU,EAAE,MAAM,CAAC;IAEnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,OAsBA;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG"}
|
package/dist/examples.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage examples for GCS store implementations
|
|
3
|
+
*/
|
|
4
|
+
import { Layer } from "effect";
|
|
5
|
+
import { GCSClientNodeJSLayer, GCSClientRESTLayer, createGCSStoreImplementation } from "./services";
|
|
6
|
+
import { UploadFileKVStore } from "@uploadista/core/types";
|
|
7
|
+
// Example 1: Node.js environment using @google-cloud/storage package
|
|
8
|
+
export function createNodeJSGCSStore(config) {
|
|
9
|
+
const gcsConfig = {
|
|
10
|
+
bucket: config.bucketName,
|
|
11
|
+
keyFilename: config.keyFilename,
|
|
12
|
+
credentials: config.credentials,
|
|
13
|
+
projectId: config.projectId,
|
|
14
|
+
};
|
|
15
|
+
return createGCSStoreImplementation({
|
|
16
|
+
...gcsConfig,
|
|
17
|
+
kvStore: {} // Will be provided by layer
|
|
18
|
+
}).pipe(Layer.provide([
|
|
19
|
+
GCSClientNodeJSLayer(gcsConfig),
|
|
20
|
+
UploadFileKVStore.Default, // or your custom KV store layer
|
|
21
|
+
]));
|
|
22
|
+
}
|
|
23
|
+
// Example 2: Cloudflare Workers environment using REST API
|
|
24
|
+
export function createCloudflareWorkerGCSStore(config) {
|
|
25
|
+
const gcsConfig = {
|
|
26
|
+
bucket: config.bucketName,
|
|
27
|
+
accessToken: config.accessToken,
|
|
28
|
+
projectId: config.projectId,
|
|
29
|
+
};
|
|
30
|
+
return createGCSStoreImplementation({
|
|
31
|
+
...gcsConfig,
|
|
32
|
+
kvStore: {} // Will be provided by layer
|
|
33
|
+
}).pipe(Layer.provide([
|
|
34
|
+
GCSClientRESTLayer(gcsConfig),
|
|
35
|
+
UploadFileKVStore.Default, // or your custom KV store layer
|
|
36
|
+
]));
|
|
37
|
+
}
|
|
38
|
+
// Example 3: Environment-aware factory
|
|
39
|
+
export function createGCSStore(config) {
|
|
40
|
+
// Detect environment
|
|
41
|
+
const isNodeJS = typeof process !== 'undefined' && process?.versions?.node;
|
|
42
|
+
if (isNodeJS && (config.keyFilename || config.credentials)) {
|
|
43
|
+
return createNodeJSGCSStore({
|
|
44
|
+
bucketName: config.bucketName,
|
|
45
|
+
keyFilename: config.keyFilename,
|
|
46
|
+
credentials: config.credentials,
|
|
47
|
+
projectId: config.projectId,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
else if (config.accessToken) {
|
|
51
|
+
return createCloudflareWorkerGCSStore({
|
|
52
|
+
bucketName: config.bucketName,
|
|
53
|
+
accessToken: config.accessToken,
|
|
54
|
+
projectId: config.projectId,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
throw new Error("Invalid configuration. Provide either (keyFilename/credentials) for Node.js or accessToken for REST API");
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Usage in your application:
|
|
63
|
+
*
|
|
64
|
+
* // Node.js
|
|
65
|
+
* const nodeJSStore = createNodeJSGCSStore({
|
|
66
|
+
* bucketName: "my-bucket",
|
|
67
|
+
* keyFilename: "/path/to/service-account.json"
|
|
68
|
+
* });
|
|
69
|
+
*
|
|
70
|
+
* // Cloudflare Workers
|
|
71
|
+
* const workersStore = createCloudflareWorkerGCSStore({
|
|
72
|
+
* bucketName: "my-bucket",
|
|
73
|
+
* accessToken: "ya29.xxx"
|
|
74
|
+
* });
|
|
75
|
+
*
|
|
76
|
+
* // Environment-aware
|
|
77
|
+
* const store = createGCSStore({
|
|
78
|
+
* bucketName: "my-bucket",
|
|
79
|
+
* keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, // Node.js
|
|
80
|
+
* accessToken: process.env.GCS_ACCESS_TOKEN, // Workers
|
|
81
|
+
* });
|
|
82
|
+
*/
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type DataStore, type KvStore, type UploadFile, UploadFileDataStore, UploadFileKVStore } from "@uploadista/core/types";
|
|
2
|
+
import { Effect, Layer } from "effect";
|
|
3
|
+
export type GCSStoreRestOptions = {
|
|
4
|
+
bucketName: string;
|
|
5
|
+
accessToken: string;
|
|
6
|
+
projectId: string;
|
|
7
|
+
kvStore: KvStore<UploadFile>;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Google Cloud Storage implementation using REST API
|
|
11
|
+
* Compatible with Cloudflare Workers
|
|
12
|
+
*/
|
|
13
|
+
export declare function gcsStoreRest({ bucketName, accessToken, projectId, kvStore, }: GCSStoreRestOptions): DataStore<UploadFile>;
|
|
14
|
+
export declare function createGCSStoreRest({ bucketName, accessToken, projectId, }: Omit<GCSStoreRestOptions, "kvStore">): Effect.Effect<DataStore<UploadFile>, never, UploadFileKVStore>;
|
|
15
|
+
export declare const GCSStoreRestLayer: (options: Omit<GCSStoreRestOptions, "kvStore">) => Layer.Layer<UploadFileDataStore, never, UploadFileKVStore>;
|
|
16
|
+
//# sourceMappingURL=gcs-store-rest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gcs-store-rest.d.ts","sourceRoot":"","sources":["../src/gcs-store-rest.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,SAAS,EAGd,KAAK,OAAO,EACZ,KAAK,UAAU,EACf,mBAAmB,EACnB,iBAAiB,EAElB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAU,MAAM,QAAQ,CAAC;AAE/C,MAAM,MAAM,mBAAmB,GAAG;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAC9B,CAAC;AAEF;;;GAGG;AACH,wBAAgB,YAAY,CAAC,EAC3B,UAAU,EACV,WAAW,EACX,SAAS,EACT,OAAO,GACR,EAAE,mBAAmB,GAAG,SAAS,CAAC,UAAU,CAAC,CAkO7C;AAED,wBAAgB,kBAAkB,CAAC,EACjC,UAAU,EACV,WAAW,EACX,SAAS,GACV,EAAE,IAAI,CAAC,mBAAmB,EAAE,SAAS,CAAC,kEAKtC;AAED,eAAO,MAAM,iBAAiB,GAC5B,SAAS,IAAI,CAAC,mBAAmB,EAAE,SAAS,CAAC,+DACoB,CAAC"}
|