@gokiteam/goki-dev 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +478 -0
  2. package/bin/goki-dev.js +452 -0
  3. package/bin/mcp-server.js +16 -0
  4. package/bin/secrets-cli.js +302 -0
  5. package/cli/ComposeOverrideGenerator.js +226 -0
  6. package/cli/ComposeParser.js +73 -0
  7. package/cli/ConfigGenerator.js +304 -0
  8. package/cli/ConfigManager.js +46 -0
  9. package/cli/DatabaseManager.js +94 -0
  10. package/cli/DevToolsChecker.js +21 -0
  11. package/cli/DevToolsDir.js +66 -0
  12. package/cli/DevToolsManager.js +451 -0
  13. package/cli/DockerManager.js +138 -0
  14. package/cli/FunctionManager.js +95 -0
  15. package/cli/HttpProxyRewriter.js +91 -0
  16. package/cli/Logger.js +10 -0
  17. package/cli/McpConfigManager.js +123 -0
  18. package/cli/NgrokManager.js +431 -0
  19. package/cli/ProjectCLI.js +2322 -0
  20. package/cli/PubSubManager.js +129 -0
  21. package/cli/SnapshotManager.js +88 -0
  22. package/cli/UiFormatter.js +292 -0
  23. package/cli/WebhookUrlRewriter.js +32 -0
  24. package/cli/secrets/BiometricAuth.js +125 -0
  25. package/cli/secrets/SecretInjector.js +47 -0
  26. package/cli/secrets/SecretsConfig.js +141 -0
  27. package/cli/secrets/SecretsDoctor.js +384 -0
  28. package/cli/secrets/SecretsManager.js +255 -0
  29. package/client/dist/client.d.ts +332 -0
  30. package/client/dist/client.js +507 -0
  31. package/client/dist/helpers.d.ts +62 -0
  32. package/client/dist/helpers.js +122 -0
  33. package/client/dist/index.d.ts +59 -0
  34. package/client/dist/index.js +78 -0
  35. package/client/dist/package.json +1 -0
  36. package/client/dist/types.d.ts +280 -0
  37. package/client/dist/types.js +7 -0
  38. package/config.development +46 -0
  39. package/config.test +18 -0
  40. package/guidelines/CodingStyleGuideline.md +148 -0
  41. package/guidelines/CommentingGuideline.md +10 -0
  42. package/guidelines/HttpApiImplementationGuideline.md +137 -0
  43. package/guidelines/NamingGuideline.md +182 -0
  44. package/package.json +138 -0
  45. package/patterns/api/[collectionName]/Controllers.md +62 -0
  46. package/patterns/api/[collectionName]/Logic.md +154 -0
  47. package/patterns/api/[collectionName]/Permissions.md +81 -0
  48. package/patterns/api/[collectionName]/Router.md +83 -0
  49. package/patterns/api/[collectionName]/Schemas.md +197 -0
  50. package/patterns/configs/Patterns.md +7 -0
  51. package/patterns/enums/Patterns.md +24 -0
  52. package/patterns/errorHandling/Patterns.md +185 -0
  53. package/patterns/testing/Patterns.md +232 -0
  54. package/src/Server.js +238 -0
  55. package/src/api/dashboard/Controllers.js +9 -0
  56. package/src/api/dashboard/Logic.js +76 -0
  57. package/src/api/dashboard/Router.js +11 -0
  58. package/src/api/dashboard/Schemas.js +47 -0
  59. package/src/api/data/Controllers.js +26 -0
  60. package/src/api/data/Logic.js +188 -0
  61. package/src/api/data/Router.js +16 -0
  62. package/src/api/docker/Controllers.js +33 -0
  63. package/src/api/docker/Logic.js +268 -0
  64. package/src/api/docker/Router.js +15 -0
  65. package/src/api/docker/Schemas.js +80 -0
  66. package/src/api/docs/Controllers.js +15 -0
  67. package/src/api/docs/Logic.js +85 -0
  68. package/src/api/docs/Router.js +12 -0
  69. package/src/api/export/Controllers.js +30 -0
  70. package/src/api/export/Logic.js +143 -0
  71. package/src/api/export/Router.js +18 -0
  72. package/src/api/export/Schemas.js +104 -0
  73. package/src/api/firestore/Controllers.js +152 -0
  74. package/src/api/firestore/Logic.js +474 -0
  75. package/src/api/firestore/Router.js +23 -0
  76. package/src/api/functions/Controllers.js +261 -0
  77. package/src/api/functions/Logic.js +710 -0
  78. package/src/api/functions/Router.js +50 -0
  79. package/src/api/functions/Schemas.js +193 -0
  80. package/src/api/gateway/Controllers.js +72 -0
  81. package/src/api/gateway/Logic.js +74 -0
  82. package/src/api/gateway/Router.js +10 -0
  83. package/src/api/gateway/Schemas.js +19 -0
  84. package/src/api/health/Controllers.js +14 -0
  85. package/src/api/health/Logic.js +24 -0
  86. package/src/api/health/Router.js +12 -0
  87. package/src/api/httpTraffic/Controllers.js +29 -0
  88. package/src/api/httpTraffic/Logic.js +33 -0
  89. package/src/api/httpTraffic/Router.js +9 -0
  90. package/src/api/httpTraffic/Schemas.js +23 -0
  91. package/src/api/logging/Controllers.js +80 -0
  92. package/src/api/logging/Logic.js +461 -0
  93. package/src/api/logging/Router.js +24 -0
  94. package/src/api/logging/Schemas.js +43 -0
  95. package/src/api/mqtt/Controllers.js +17 -0
  96. package/src/api/mqtt/Logic.js +66 -0
  97. package/src/api/mqtt/Router.js +12 -0
  98. package/src/api/postgres/Controllers.js +97 -0
  99. package/src/api/postgres/Logic.js +221 -0
  100. package/src/api/postgres/Router.js +21 -0
  101. package/src/api/pubsub/Controllers.js +236 -0
  102. package/src/api/pubsub/Logic.js +732 -0
  103. package/src/api/pubsub/Router.js +41 -0
  104. package/src/api/pubsub/Schemas.js +355 -0
  105. package/src/api/redis/Controllers.js +63 -0
  106. package/src/api/redis/Logic.js +239 -0
  107. package/src/api/redis/Router.js +21 -0
  108. package/src/api/scheduler/Controllers.js +27 -0
  109. package/src/api/scheduler/Logic.js +49 -0
  110. package/src/api/scheduler/Router.js +16 -0
  111. package/src/api/services/Controllers.js +26 -0
  112. package/src/api/services/Logic.js +205 -0
  113. package/src/api/services/Router.js +14 -0
  114. package/src/api/services/Schemas.js +66 -0
  115. package/src/api/snapshots/Controllers.js +37 -0
  116. package/src/api/snapshots/Logic.js +797 -0
  117. package/src/api/snapshots/Router.js +15 -0
  118. package/src/api/snapshots/Schemas.js +23 -0
  119. package/src/api/webhooks/Controllers.js +49 -0
  120. package/src/api/webhooks/Logic.js +137 -0
  121. package/src/api/webhooks/Router.js +12 -0
  122. package/src/api/webhooks/Schemas.js +31 -0
  123. package/src/configs/Application.js +147 -0
  124. package/src/configs/Default.js +13 -0
  125. package/src/consumers/BlackboxLogsConsumer.js +235 -0
  126. package/src/consumers/DockerLogsConsumer.js +687 -0
  127. package/src/db/Tables.js +66 -0
  128. package/src/db/schemas/firestore.js +18 -0
  129. package/src/db/schemas/functions.js +65 -0
  130. package/src/db/schemas/httpTraffic.js +43 -0
  131. package/src/db/schemas/logging.js +74 -0
  132. package/src/db/schemas/migrations.js +64 -0
  133. package/src/db/schemas/mqtt.js +56 -0
  134. package/src/db/schemas/pubsub.js +90 -0
  135. package/src/db/schemas/pubsubRegistry.js +22 -0
  136. package/src/db/schemas/webhooks.js +28 -0
  137. package/src/emulation/awsiot/Controllers.js +91 -0
  138. package/src/emulation/awsiot/Logic.js +70 -0
  139. package/src/emulation/awsiot/Router.js +19 -0
  140. package/src/emulation/awsiot/Server.js +100 -0
  141. package/src/emulation/firestore/Server.js +136 -0
  142. package/src/emulation/logging/Controllers.js +212 -0
  143. package/src/emulation/logging/Logic.js +416 -0
  144. package/src/emulation/logging/Router.js +36 -0
  145. package/src/emulation/logging/Schemas.js +82 -0
  146. package/src/emulation/logging/Server.js +108 -0
  147. package/src/emulation/pubsub/Controllers.js +279 -0
  148. package/src/emulation/pubsub/DefaultTopics.js +162 -0
  149. package/src/emulation/pubsub/Logic.js +427 -0
  150. package/src/emulation/pubsub/README.md +309 -0
  151. package/src/emulation/pubsub/Router.js +33 -0
  152. package/src/emulation/pubsub/Server.js +104 -0
  153. package/src/emulation/pubsub/ShadowPoller.js +276 -0
  154. package/src/emulation/pubsub/ShadowSubscriptionManager.js +199 -0
  155. package/src/enums/ContainerNames.js +106 -0
  156. package/src/enums/ErrorReason.js +28 -0
  157. package/src/enums/FunctionStatuses.js +15 -0
  158. package/src/enums/FunctionTriggerTypes.js +15 -0
  159. package/src/enums/GatewayState.js +7 -0
  160. package/src/enums/ServiceNames.js +68 -0
  161. package/src/jobs/DatabaseMaintenance.js +184 -0
  162. package/src/jobs/MessageHistoryCleanup.js +152 -0
  163. package/src/mcp/ApiClient.js +25 -0
  164. package/src/mcp/Server.js +52 -0
  165. package/src/mcp/prompts/debugging.js +104 -0
  166. package/src/mcp/resources/platform.js +118 -0
  167. package/src/mcp/tools/data.js +84 -0
  168. package/src/mcp/tools/docker.js +166 -0
  169. package/src/mcp/tools/firestore.js +162 -0
  170. package/src/mcp/tools/functions.js +380 -0
  171. package/src/mcp/tools/httpTraffic.js +69 -0
  172. package/src/mcp/tools/logging.js +174 -0
  173. package/src/mcp/tools/mqtt.js +37 -0
  174. package/src/mcp/tools/postgres.js +130 -0
  175. package/src/mcp/tools/pubsub.js +316 -0
  176. package/src/mcp/tools/redis.js +146 -0
  177. package/src/mcp/tools/services.js +169 -0
  178. package/src/mcp/tools/snapshots.js +88 -0
  179. package/src/mcp/tools/webhooks.js +115 -0
  180. package/src/middleware/DevProxy.js +67 -0
  181. package/src/middleware/ErrorCatcher.js +35 -0
  182. package/src/middleware/HttpProxy.js +215 -0
  183. package/src/middleware/Reply.js +24 -0
  184. package/src/middleware/TraceId.js +9 -0
  185. package/src/middleware/WebhookProxy.js +234 -0
  186. package/src/protocols/mqtt/Broker.js +92 -0
  187. package/src/protocols/mqtt/Handlers.js +175 -0
  188. package/src/protocols/mqtt/PubSubBridge.js +162 -0
  189. package/src/protocols/mqtt/Server.js +116 -0
  190. package/src/runtime/FunctionRunner.js +179 -0
  191. package/src/services/AppGatewayService.js +582 -0
  192. package/src/singletons/FirestoreBroadcaster.js +367 -0
  193. package/src/singletons/FunctionTriggerDispatcher.js +456 -0
  194. package/src/singletons/FunctionsService.js +418 -0
  195. package/src/singletons/HttpProxy.js +224 -0
  196. package/src/singletons/LogBroadcaster.js +159 -0
  197. package/src/singletons/Logger.js +49 -0
  198. package/src/singletons/MemoryJsonStore.js +175 -0
  199. package/src/singletons/MessageBroadcaster.js +190 -0
  200. package/src/singletons/PostgresBroadcaster.js +367 -0
  201. package/src/singletons/PostgresClient.js +180 -0
  202. package/src/singletons/RedisClient.js +184 -0
  203. package/src/singletons/SqliteStore.js +480 -0
  204. package/src/singletons/TickService.js +151 -0
  205. package/src/singletons/WebhookProxy.js +223 -0
@@ -0,0 +1,797 @@
1
+ import { execa } from 'execa'
2
+ import fs from 'fs/promises'
3
+ import path from 'path'
4
+ import { fileURLToPath } from 'url'
5
+ import { Logic as FirestoreLogic } from '../firestore/Logic.js'
6
+ import { Logic as DockerLogic } from '../docker/Logic.js'
7
+ import { Logic as PubSubLogic } from '../pubsub/Logic.js'
8
+ import { Logger } from '../../singletons/Logger.js'
9
+ import { Application } from '../../configs/Application.js'
10
+ import { SqliteStore } from '../../singletons/SqliteStore.js'
11
+ import {
12
+ PUBSUB_TOPICS,
13
+ PUBSUB_SUBSCRIPTIONS,
14
+ PUBSUB_TOPIC_REGISTRY,
15
+ FIRESTORE_METADATA,
16
+ WEBHOOK_ROUTES,
17
+ WEBHOOK_SETTINGS,
18
+ LOGGING_ENTRIES,
19
+ LOGGING_LOGS,
20
+ HTTP_TRAFFIC,
21
+ MQTT_MESSAGES,
22
+ PUBSUB_MESSAGES,
23
+ PUBSUB_MESSAGE_HISTORY
24
+ } from '../../db/Tables.js'
25
+
26
+ const __filename = fileURLToPath(import.meta.url)
27
+ const __dirname = path.dirname(__filename)
28
+ const PROJECT_ROOT = path.join(__dirname, '../../..')
29
+ const SNAPSHOTS_DIR = path.join(PROJECT_ROOT, '.goki-dev/snapshots')
30
+
31
+ // Ensure snapshot directories exist
32
+ async function ensureSnapshotDirs () {
33
+ await fs.mkdir(path.join(SNAPSHOTS_DIR, 'postgres'), { recursive: true })
34
+ await fs.mkdir(path.join(SNAPSHOTS_DIR, 'firestore'), { recursive: true })
35
+ await fs.mkdir(path.join(SNAPSHOTS_DIR, 'sqlite'), { recursive: true })
36
+ }
37
+
38
+ // Configuration tables to snapshot (important metadata)
39
+ const CONFIG_TABLES = [
40
+ PUBSUB_TOPICS,
41
+ PUBSUB_SUBSCRIPTIONS,
42
+ PUBSUB_TOPIC_REGISTRY,
43
+ FIRESTORE_METADATA,
44
+ WEBHOOK_ROUTES,
45
+ WEBHOOK_SETTINGS
46
+ ]
47
+
48
+ // Log/traffic tables to clear (not restore - fresh start for tests)
49
+ const LOG_TABLES = [
50
+ LOGGING_ENTRIES,
51
+ LOGGING_LOGS,
52
+ HTTP_TRAFFIC,
53
+ MQTT_MESSAGES,
54
+ PUBSUB_MESSAGES,
55
+ PUBSUB_MESSAGE_HISTORY
56
+ ]
57
+
58
+ // Generate snapshot ID
59
+ function generateSnapshotId () {
60
+ const now = new Date()
61
+ const timestamp = now.toISOString().replace(/[:.]/g, '-').replace('Z', '').replace('T', '_')
62
+ return `e2e_${timestamp}`
63
+ }
64
+
65
+ // Get snapshot metadata file path
66
+ function getMetadataPath () {
67
+ return path.join(SNAPSHOTS_DIR, 'metadata.json')
68
+ }
69
+
70
+ // Read metadata
71
+ async function readMetadata () {
72
+ try {
73
+ const data = await fs.readFile(getMetadataPath(), 'utf8')
74
+ return JSON.parse(data)
75
+ } catch (error) {
76
+ return { snapshots: [] }
77
+ }
78
+ }
79
+
80
+ // Write metadata
81
+ async function writeMetadata (metadata) {
82
+ await fs.writeFile(getMetadataPath(), JSON.stringify(metadata, null, 2))
83
+ }
84
+
85
+ // Get file size
86
+ async function getFileSize (filePath) {
87
+ try {
88
+ const stats = await fs.stat(filePath)
89
+ return stats.size
90
+ } catch {
91
+ return 0
92
+ }
93
+ }
94
+
95
+ // PostgreSQL snapshot functions
96
+ async function createPostgresSnapshot (database, snapshotId, traceId) {
97
+ const dumpPath = path.join(SNAPSHOTS_DIR, 'postgres', `${snapshotId}.dump`)
98
+ try {
99
+ Logger.log({
100
+ level: 'info',
101
+ message: 'Creating PostgreSQL snapshot',
102
+ data: { database, snapshotId, traceId }
103
+ })
104
+ await execa('pg_dump', [
105
+ '-h', Application.postgres.host,
106
+ '-p', String(Application.postgres.port),
107
+ '-U', Application.postgres.user,
108
+ '-F', 'c', // custom format
109
+ '-f', dumpPath,
110
+ database
111
+ ], {
112
+ env: { PGPASSWORD: Application.postgres.password }
113
+ })
114
+ const size = await getFileSize(dumpPath)
115
+ Logger.log({
116
+ level: 'info',
117
+ message: 'PostgreSQL snapshot created',
118
+ data: { database, size, traceId }
119
+ })
120
+ return { path: dumpPath, size, database }
121
+ } catch (error) {
122
+ Logger.log({
123
+ level: 'error',
124
+ message: 'Failed to create PostgreSQL snapshot',
125
+ data: { database, error: error.message, traceId }
126
+ })
127
+ throw new Error(`PostgreSQL snapshot failed: ${error.message}`)
128
+ }
129
+ }
130
+
131
+ async function restorePostgresSnapshot (database, snapshotId, traceId) {
132
+ const dumpPath = path.join(SNAPSHOTS_DIR, 'postgres', `${snapshotId}.dump`)
133
+ try {
134
+ Logger.log({
135
+ level: 'info',
136
+ message: 'Restoring PostgreSQL snapshot',
137
+ data: { database, snapshotId, traceId }
138
+ })
139
+ await execa('pg_restore', [
140
+ '--clean',
141
+ '--if-exists',
142
+ '-h', Application.postgres.host,
143
+ '-p', String(Application.postgres.port),
144
+ '-U', Application.postgres.user,
145
+ '-d', database,
146
+ dumpPath
147
+ ], {
148
+ env: { PGPASSWORD: Application.postgres.password }
149
+ })
150
+ Logger.log({
151
+ level: 'info',
152
+ message: 'PostgreSQL snapshot restored',
153
+ data: { database, traceId }
154
+ })
155
+ } catch (error) {
156
+ Logger.log({
157
+ level: 'error',
158
+ message: 'Failed to restore PostgreSQL snapshot',
159
+ data: { database, error: error.message, traceId }
160
+ })
161
+ throw new Error(`PostgreSQL restore failed: ${error.message}`)
162
+ }
163
+ }
164
+
165
+ // Firestore snapshot functions
166
+ async function createFirestoreSnapshot (snapshotId, traceId) {
167
+ const exportPath = path.join(SNAPSHOTS_DIR, 'firestore', `${snapshotId}.json`)
168
+ try {
169
+ Logger.log({
170
+ level: 'info',
171
+ message: 'Creating Firestore snapshot',
172
+ data: { snapshotId, traceId }
173
+ })
174
+ const collectionsResult = await FirestoreLogic.listCollections({ traceId })
175
+ const collections = collectionsResult.collections || []
176
+ const exportData = {}
177
+ let totalDocuments = 0
178
+ // Export collections in parallel for better performance
179
+ await Promise.all(collections.map(async (collection) => {
180
+ const docsResult = await FirestoreLogic.listDocuments({
181
+ collectionPath: collection,
182
+ page: { limit: 10000 },
183
+ traceId
184
+ })
185
+ const documents = docsResult.documents || []
186
+ exportData[collection] = documents
187
+ totalDocuments += documents.length
188
+ }))
189
+ await fs.writeFile(exportPath, JSON.stringify(exportData, null, 2))
190
+ const size = await getFileSize(exportPath)
191
+ Logger.log({
192
+ level: 'info',
193
+ message: 'Firestore snapshot created',
194
+ data: { collections: collections.length, documents: totalDocuments, size, traceId }
195
+ })
196
+ return { path: exportPath, collections: collections.length, documents: totalDocuments, size }
197
+ } catch (error) {
198
+ Logger.log({
199
+ level: 'error',
200
+ message: 'Failed to create Firestore snapshot',
201
+ data: { error: error.message, traceId }
202
+ })
203
+ throw new Error(`Firestore snapshot failed: ${error.message}`)
204
+ }
205
+ }
206
+
207
+ async function restoreFirestoreSnapshot (snapshotId, traceId) {
208
+ const exportPath = path.join(SNAPSHOTS_DIR, 'firestore', `${snapshotId}.json`)
209
+ try {
210
+ Logger.log({
211
+ level: 'info',
212
+ message: 'Restoring Firestore snapshot',
213
+ data: { snapshotId, traceId }
214
+ })
215
+ const data = await fs.readFile(exportPath, 'utf8')
216
+ const exportData = JSON.parse(data)
217
+ // Clear all collections first
218
+ for (const collection in exportData) {
219
+ await FirestoreLogic.clearCollection({ collectionPath: collection, traceId })
220
+ }
221
+ // Restore documents in parallel batches for better performance
222
+ let restoredCount = 0
223
+ const BATCH_SIZE = 20
224
+ for (const collection in exportData) {
225
+ const documents = exportData[collection]
226
+ // Process documents in batches of 20 in parallel
227
+ for (let i = 0; i < documents.length; i += BATCH_SIZE) {
228
+ const batch = documents.slice(i, i + BATCH_SIZE)
229
+ await Promise.all(batch.map(async (doc) => {
230
+ const docId = doc.name.split('/').pop()
231
+ const fields = doc.fields || {}
232
+ await FirestoreLogic.createDocument({
233
+ collectionPath: collection,
234
+ documentId: docId,
235
+ fields,
236
+ traceId
237
+ })
238
+ }))
239
+ restoredCount += batch.length
240
+ }
241
+ }
242
+ Logger.log({
243
+ level: 'info',
244
+ message: 'Firestore snapshot restored',
245
+ data: { collections: Object.keys(exportData).length, documents: restoredCount, traceId }
246
+ })
247
+ } catch (error) {
248
+ Logger.log({
249
+ level: 'error',
250
+ message: 'Failed to restore Firestore snapshot',
251
+ data: { error: error.message, traceId }
252
+ })
253
+ throw new Error(`Firestore restore failed: ${error.message}`)
254
+ }
255
+ }
256
+
257
+ // Redis is in-memory only, no need to snapshot
258
+
259
+ // SQLite snapshot functions
260
+ async function createSqliteSnapshot (snapshotId, traceId) {
261
+ const exportPath = path.join(SNAPSHOTS_DIR, 'sqlite', `${snapshotId}.json`)
262
+ try {
263
+ Logger.log({
264
+ level: 'info',
265
+ message: 'Creating SQLite snapshot',
266
+ data: { snapshotId, traceId }
267
+ })
268
+ const exportData = {}
269
+ let totalRecords = 0
270
+ let snapshotedTables = 0
271
+ // Snapshot config tables (skip if table doesn't exist)
272
+ for (const tableName of CONFIG_TABLES) {
273
+ try {
274
+ const result = SqliteStore.list(tableName)
275
+ exportData[tableName] = result.data || []
276
+ totalRecords += exportData[tableName].length
277
+ snapshotedTables++
278
+ } catch (error) {
279
+ // Table doesn't exist - skip it
280
+ Logger.log({
281
+ level: 'debug',
282
+ message: 'Skipping non-existent table',
283
+ data: { tableName, traceId }
284
+ })
285
+ }
286
+ }
287
+ // Also export Pub/Sub data using internal APIs
288
+ const pubsubData = await exportPubSubData(traceId)
289
+ exportData._pubsub_data = pubsubData
290
+ totalRecords += pubsubData.topics.length + pubsubData.subscriptions.length
291
+ await fs.writeFile(exportPath, JSON.stringify(exportData, null, 2))
292
+ const size = await getFileSize(exportPath)
293
+ Logger.log({
294
+ level: 'info',
295
+ message: 'SQLite snapshot created',
296
+ data: { tables: snapshotedTables, records: totalRecords, pubsubTopics: pubsubData.topics.length, pubsubSubs: pubsubData.subscriptions.length, size, traceId }
297
+ })
298
+ return { path: exportPath, tables: snapshotedTables, records: totalRecords, size }
299
+ } catch (error) {
300
+ Logger.log({
301
+ level: 'error',
302
+ message: 'Failed to create SQLite snapshot',
303
+ data: { error: error.message, traceId }
304
+ })
305
+ throw new Error(`SQLite snapshot failed: ${error.message}`)
306
+ }
307
+ }
308
+
309
+ async function restoreSqliteSnapshot (snapshotId, traceId) {
310
+ const exportPath = path.join(SNAPSHOTS_DIR, 'sqlite', `${snapshotId}.json`)
311
+ try {
312
+ Logger.log({
313
+ level: 'info',
314
+ message: 'Restoring SQLite snapshot',
315
+ data: { snapshotId, traceId }
316
+ })
317
+ const data = await fs.readFile(exportPath, 'utf8')
318
+ const exportData = JSON.parse(data)
319
+ // Clear config tables first (delete all records)
320
+ for (const tableName of CONFIG_TABLES) {
321
+ try {
322
+ // Get all records and delete by ID to ensure they're gone
323
+ const result = SqliteStore.list(tableName)
324
+ const records = result.data || []
325
+ for (const record of records) {
326
+ try {
327
+ SqliteStore.delete(tableName, record._id)
328
+ } catch (err) {
329
+ // Ignore delete errors
330
+ }
331
+ }
332
+ } catch (error) {
333
+ // Table doesn't exist - skip
334
+ }
335
+ }
336
+ // Restore config tables (skip _pubsub_data which is not a table)
337
+ let restoredCount = 0
338
+ for (const tableName in exportData) {
339
+ // Skip special fields that aren't SQLite tables
340
+ if (tableName.startsWith('_')) continue
341
+ const records = exportData[tableName] || []
342
+ if (!Array.isArray(records)) continue
343
+ for (const record of records) {
344
+ try {
345
+ // Remove _id field before creating (let SQLite generate new IDs)
346
+ const { _id, ...recordData } = record
347
+ SqliteStore.create(tableName, recordData)
348
+ restoredCount++
349
+ } catch (error) {
350
+ // Skip duplicates or other errors
351
+ if (!error.message.includes('UNIQUE constraint')) {
352
+ Logger.log({
353
+ level: 'warn',
354
+ message: 'Failed to restore record',
355
+ data: { tableName, error: error.message, traceId }
356
+ })
357
+ }
358
+ }
359
+ }
360
+ }
361
+ // Clear log/traffic tables (fresh start for tests, skip if don't exist)
362
+ for (const tableName of LOG_TABLES) {
363
+ try {
364
+ SqliteStore.deleteWhere(tableName, {})
365
+ } catch (error) {
366
+ // Table doesn't exist - skip
367
+ }
368
+ }
369
+ Logger.log({
370
+ level: 'info',
371
+ message: 'SQLite snapshot restored',
372
+ data: { records: restoredCount, traceId }
373
+ })
374
+ // Return Pub/Sub data for re-registration
375
+ return exportData._pubsub_data || { topics: [], subscriptions: [] }
376
+ } catch (error) {
377
+ Logger.log({
378
+ level: 'error',
379
+ message: 'Failed to restore SQLite snapshot',
380
+ data: { error: error.message, traceId }
381
+ })
382
+ throw new Error(`SQLite restore failed: ${error.message}`)
383
+ }
384
+ }
385
+
386
+ // Export Pub/Sub data directly from emulator (source of truth)
387
+ async function exportPubSubData (traceId) {
388
+ const PUBSUB_API = `http://${Application.pubsub.emulatorHost}:${Application.pubsub.emulatorPort}`
389
+ const PROJECT_ID = Application.pubsub.projectId
390
+ const exportData = { topics: [], subscriptions: [] }
391
+ try {
392
+ // Get all topics from emulator
393
+ const topicsRes = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics`)
394
+ if (topicsRes.ok) {
395
+ const topicsData = await topicsRes.json()
396
+ exportData.topics = topicsData.topics || []
397
+ }
398
+ // Get all subscriptions from emulator
399
+ const subsRes = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/subscriptions`)
400
+ if (subsRes.ok) {
401
+ const subsData = await subsRes.json()
402
+ exportData.subscriptions = subsData.subscriptions || []
403
+ }
404
+ Logger.log({
405
+ level: 'info',
406
+ message: 'Pub/Sub data exported from emulator',
407
+ data: { topics: exportData.topics.length, subscriptions: exportData.subscriptions.length, traceId }
408
+ })
409
+ return exportData
410
+ } catch (error) {
411
+ Logger.log({
412
+ level: 'warn',
413
+ message: 'Failed to export Pub/Sub data from emulator',
414
+ data: { error: error.message, traceId }
415
+ })
416
+ return exportData
417
+ }
418
+ }
419
+
420
+ // Restore Pub/Sub topics and subscriptions directly to emulator
421
+ async function restorePubSubData (pubsubData, traceId) {
422
+ const PUBSUB_API = `http://${Application.pubsub.emulatorHost}:${Application.pubsub.emulatorPort}`
423
+ try {
424
+ Logger.log({
425
+ level: 'info',
426
+ message: 'Restoring Pub/Sub data to emulator',
427
+ data: { topics: pubsubData.topics?.length || 0, subscriptions: pubsubData.subscriptions?.length || 0, traceId }
428
+ })
429
+ // Re-create topics via emulator HTTP API
430
+ for (const topic of (pubsubData.topics || [])) {
431
+ try {
432
+ await fetch(`${PUBSUB_API}/v1/${topic.name}`, {
433
+ method: 'PUT',
434
+ headers: { 'Content-Type': 'application/json' },
435
+ body: JSON.stringify({ name: topic.name, labels: topic.labels || {} })
436
+ })
437
+ } catch (error) {
438
+ // Ignore errors (topic may already exist)
439
+ }
440
+ }
441
+ // Re-create subscriptions via emulator HTTP API
442
+ for (const sub of (pubsubData.subscriptions || [])) {
443
+ try {
444
+ await fetch(`${PUBSUB_API}/v1/${sub.name}`, {
445
+ method: 'PUT',
446
+ headers: { 'Content-Type': 'application/json' },
447
+ body: JSON.stringify({
448
+ name: sub.name,
449
+ topic: sub.topic,
450
+ pushConfig: sub.pushConfig || {},
451
+ ackDeadlineSeconds: sub.ackDeadlineSeconds || 10
452
+ })
453
+ })
454
+ } catch (error) {
455
+ // Ignore errors (subscription may already exist)
456
+ }
457
+ }
458
+ Logger.log({
459
+ level: 'info',
460
+ message: 'Pub/Sub data restored to emulator',
461
+ data: { traceId }
462
+ })
463
+ } catch (error) {
464
+ Logger.log({
465
+ level: 'error',
466
+ message: 'Failed to restore Pub/Sub data to emulator',
467
+ data: { error: error.message, traceId }
468
+ })
469
+ throw new Error(`Pub/Sub restore failed: ${error.message}`)
470
+ }
471
+ }
472
+
473
+ // Get microservice containers (excluding infrastructure)
474
+ async function getMicroserviceContainers (traceId) {
475
+ const result = await DockerLogic.listContainers({ traceId })
476
+ const containers = result.data || []
477
+ // Filter for microservices only (not infrastructure)
478
+ return containers.filter(c => c.category === 'microservice' && c.status === 'running')
479
+ }
480
+
481
+ // Stop microservices
482
+ async function stopMicroservices (traceId) {
483
+ const containers = await getMicroserviceContainers(traceId)
484
+ Logger.log({
485
+ level: 'info',
486
+ message: 'Stopping microservices',
487
+ data: { count: containers.length, traceId }
488
+ })
489
+ for (const container of containers) {
490
+ await DockerLogic.stopContainer({ containerName: container.name, traceId })
491
+ }
492
+ return containers.map(c => c.name)
493
+ }
494
+
495
+ // Start microservices
496
+ async function startMicroservices (containerNames, traceId) {
497
+ Logger.log({
498
+ level: 'info',
499
+ message: 'Starting microservices',
500
+ data: { count: containerNames.length, traceId }
501
+ })
502
+ for (const containerName of containerNames) {
503
+ await DockerLogic.startContainer({ containerName, traceId })
504
+ }
505
+ }
506
+
507
+ export const Logic = {
508
+ async createSnapshot (params) {
509
+ const { services = [], traceId } = params
510
+ const snapshotId = generateSnapshotId()
511
+ const timestamp = new Date().toISOString()
512
+ const snapshotData = {
513
+ id: snapshotId,
514
+ timestamp,
515
+ services: {}
516
+ }
517
+ try {
518
+ await ensureSnapshotDirs()
519
+ // Determine which services to snapshot
520
+ const snapshotPostgres = services.length === 0 || services.includes('postgres')
521
+ const snapshotFirestore = services.length === 0 || services.includes('firestore')
522
+ const snapshotSqlite = services.length === 0 || services.includes('sqlite')
523
+ // Create snapshots (only if service is configured and reachable)
524
+ if (snapshotPostgres && Application.postgres.database) {
525
+ try {
526
+ snapshotData.services.postgres = await createPostgresSnapshot(
527
+ Application.postgres.database,
528
+ snapshotId,
529
+ traceId
530
+ )
531
+ } catch (error) {
532
+ // PostgreSQL not available - skip with warning
533
+ Logger.log({
534
+ level: 'warn',
535
+ message: 'PostgreSQL snapshot skipped (service not available)',
536
+ data: { error: error.message, traceId }
537
+ })
538
+ }
539
+ }
540
+ if (snapshotFirestore) {
541
+ snapshotData.services.firestore = await createFirestoreSnapshot(snapshotId, traceId)
542
+ }
543
+ if (snapshotSqlite) {
544
+ snapshotData.services.sqlite = await createSqliteSnapshot(snapshotId, traceId)
545
+ }
546
+ // Save metadata
547
+ const metadata = await readMetadata()
548
+ metadata.snapshots.push(snapshotData)
549
+ await writeMetadata(metadata)
550
+ Logger.log({
551
+ level: 'info',
552
+ message: 'Snapshot created successfully',
553
+ data: { snapshotId, traceId }
554
+ })
555
+ return {
556
+ snapshotId,
557
+ timestamp,
558
+ services: snapshotData.services,
559
+ traceId
560
+ }
561
+ } catch (error) {
562
+ Logger.log({
563
+ level: 'error',
564
+ message: 'Failed to create snapshot',
565
+ data: { error: error.message, traceId }
566
+ })
567
+ return {
568
+ status: 'error',
569
+ message: `Snapshot creation failed: ${error.message}`,
570
+ traceId
571
+ }
572
+ }
573
+ },
574
+
575
+ async restoreSnapshot (params) {
576
+ const { snapshotId, restartServices = true, traceId } = params
577
+ const restoredServices = []
578
+ const errors = []
579
+ let stoppedContainers = []
580
+ try {
581
+ const metadata = await readMetadata()
582
+ const snapshot = metadata.snapshots.find(s => s.id === snapshotId)
583
+ if (!snapshot) {
584
+ return {
585
+ status: 'error',
586
+ message: `Snapshot not found: ${snapshotId}`,
587
+ traceId
588
+ }
589
+ }
590
+ // Stop microservices first to ensure they reload data
591
+ if (restartServices) {
592
+ stoppedContainers = await stopMicroservices(traceId)
593
+ }
594
+ // Restore PostgreSQL
595
+ if (snapshot.services.postgres) {
596
+ try {
597
+ await restorePostgresSnapshot(snapshot.services.postgres.database, snapshotId, traceId)
598
+ restoredServices.push('postgres')
599
+ } catch (error) {
600
+ errors.push({ service: 'postgres', error: error.message })
601
+ }
602
+ }
603
+ // Restore Firestore
604
+ if (snapshot.services.firestore) {
605
+ try {
606
+ await restoreFirestoreSnapshot(snapshotId, traceId)
607
+ restoredServices.push('firestore')
608
+ } catch (error) {
609
+ errors.push({ service: 'firestore', error: error.message })
610
+ }
611
+ }
612
+ // Restore SQLite
613
+ let pubsubDataToRestore = null
614
+ if (snapshot.services.sqlite) {
615
+ try {
616
+ pubsubDataToRestore = await restoreSqliteSnapshot(snapshotId, traceId)
617
+ restoredServices.push('sqlite')
618
+ } catch (error) {
619
+ errors.push({ service: 'sqlite', error: error.message })
620
+ }
621
+ }
622
+ // Re-register Pub/Sub topics/subscriptions before restarting services
623
+ if (pubsubDataToRestore && (pubsubDataToRestore.topics?.length > 0 || pubsubDataToRestore.subscriptions?.length > 0)) {
624
+ try {
625
+ await restorePubSubData(pubsubDataToRestore, traceId)
626
+ } catch (error) {
627
+ errors.push({ service: 'pubsub-registration', error: error.message })
628
+ }
629
+ }
630
+ // Restart microservices (now that Pub/Sub topics are ready)
631
+ if (restartServices && stoppedContainers.length > 0) {
632
+ await startMicroservices(stoppedContainers, traceId)
633
+ }
634
+ Logger.log({
635
+ level: 'info',
636
+ message: 'Snapshot restored successfully',
637
+ data: { snapshotId, restoredServices, traceId }
638
+ })
639
+ return {
640
+ restored: true,
641
+ services: restoredServices,
642
+ errors: errors.length > 0 ? errors : undefined,
643
+ traceId
644
+ }
645
+ } catch (error) {
646
+ // Try to restart services even if restore failed
647
+ if (restartServices && stoppedContainers.length > 0) {
648
+ try {
649
+ await startMicroservices(stoppedContainers, traceId)
650
+ } catch (restartError) {
651
+ Logger.log({
652
+ level: 'error',
653
+ message: 'Failed to restart services after failed restore',
654
+ data: { error: restartError.message, traceId }
655
+ })
656
+ }
657
+ }
658
+ Logger.log({
659
+ level: 'error',
660
+ message: 'Failed to restore snapshot',
661
+ data: { error: error.message, traceId }
662
+ })
663
+ return {
664
+ status: 'error',
665
+ message: `Snapshot restore failed: ${error.message}`,
666
+ errors,
667
+ traceId
668
+ }
669
+ }
670
+ },
671
+
672
+ async listSnapshots (params) {
673
+ const { traceId } = params
674
+ try {
675
+ const metadata = await readMetadata()
676
+ return {
677
+ snapshots: metadata.snapshots.map(s => ({
678
+ id: s.id,
679
+ timestamp: s.timestamp,
680
+ services: Object.keys(s.services),
681
+ size: Object.values(s.services).reduce((sum, svc) => sum + (svc.size || 0), 0)
682
+ })),
683
+ total: metadata.snapshots.length,
684
+ traceId
685
+ }
686
+ } catch (error) {
687
+ Logger.log({
688
+ level: 'error',
689
+ message: 'Failed to list snapshots',
690
+ data: { error: error.message, traceId }
691
+ })
692
+ return {
693
+ snapshots: [],
694
+ total: 0,
695
+ traceId
696
+ }
697
+ }
698
+ },
699
+
700
+ async getSnapshotDetails (params) {
701
+ const { snapshotId, traceId } = params
702
+ try {
703
+ const metadata = await readMetadata()
704
+ const snapshot = metadata.snapshots.find(s => s.id === snapshotId)
705
+ if (!snapshot) {
706
+ return {
707
+ status: 'error',
708
+ message: `Snapshot not found: ${snapshotId}`,
709
+ traceId
710
+ }
711
+ }
712
+ return {
713
+ snapshot,
714
+ traceId
715
+ }
716
+ } catch (error) {
717
+ Logger.log({
718
+ level: 'error',
719
+ message: 'Failed to get snapshot details',
720
+ data: { error: error.message, traceId }
721
+ })
722
+ return {
723
+ status: 'error',
724
+ message: `Failed to get snapshot details: ${error.message}`,
725
+ traceId
726
+ }
727
+ }
728
+ },
729
+
730
+ async deleteSnapshot (params) {
731
+ const { snapshotId, traceId } = params
732
+ try {
733
+ const metadata = await readMetadata()
734
+ const snapshot = metadata.snapshots.find(s => s.id === snapshotId)
735
+ if (!snapshot) {
736
+ // If no specific snapshot ID, delete all
737
+ if (!snapshotId) {
738
+ // Delete all snapshot files
739
+ await fs.rm(path.join(SNAPSHOTS_DIR, 'postgres'), { recursive: true, force: true })
740
+ await fs.rm(path.join(SNAPSHOTS_DIR, 'firestore'), { recursive: true, force: true })
741
+ await fs.rm(path.join(SNAPSHOTS_DIR, 'sqlite'), { recursive: true, force: true })
742
+ await ensureSnapshotDirs()
743
+ await writeMetadata({ snapshots: [] })
744
+ Logger.log({
745
+ level: 'info',
746
+ message: 'All snapshots deleted',
747
+ data: { traceId }
748
+ })
749
+ return {
750
+ deleted: true,
751
+ message: 'All snapshots deleted',
752
+ traceId
753
+ }
754
+ }
755
+ return {
756
+ status: 'error',
757
+ message: `Snapshot not found: ${snapshotId}`,
758
+ traceId
759
+ }
760
+ }
761
+ // Delete snapshot files
762
+ if (snapshot.services.postgres) {
763
+ await fs.unlink(snapshot.services.postgres.path).catch(() => {})
764
+ }
765
+ if (snapshot.services.firestore) {
766
+ await fs.unlink(snapshot.services.firestore.path).catch(() => {})
767
+ }
768
+ if (snapshot.services.sqlite) {
769
+ await fs.unlink(snapshot.services.sqlite.path).catch(() => {})
770
+ }
771
+ // Remove from metadata
772
+ metadata.snapshots = metadata.snapshots.filter(s => s.id !== snapshotId)
773
+ await writeMetadata(metadata)
774
+ Logger.log({
775
+ level: 'info',
776
+ message: 'Snapshot deleted',
777
+ data: { snapshotId, traceId }
778
+ })
779
+ return {
780
+ deleted: true,
781
+ snapshotId,
782
+ traceId
783
+ }
784
+ } catch (error) {
785
+ Logger.log({
786
+ level: 'error',
787
+ message: 'Failed to delete snapshot',
788
+ data: { error: error.message, traceId }
789
+ })
790
+ return {
791
+ status: 'error',
792
+ message: `Failed to delete snapshot: ${error.message}`,
793
+ traceId
794
+ }
795
+ }
796
+ }
797
+ }