@restforgejs/platform 5.1.0 → 5.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/build-info.json +2 -2
  2. package/cli/consumer-deploy.js +1 -1
  3. package/cli/consumer.js +1 -1
  4. package/generators/cli/data/pull.js +104 -0
  5. package/generators/cli/data/push.js +95 -0
  6. package/generators/cli/fast-track.js +12 -25
  7. package/generators/cli/init.js +347 -97
  8. package/generators/cli/schema/introspect.js +10 -10
  9. package/generators/lib/data/data-scope.js +138 -0
  10. package/generators/lib/data/db-executor.js +440 -0
  11. package/generators/lib/data/dialect-kit.js +56 -0
  12. package/generators/lib/data/envelope.js +236 -0
  13. package/generators/lib/data/pull-runner.js +414 -0
  14. package/generators/lib/data/push-runner.js +400 -0
  15. package/generators/lib/data/sdf-reader.js +132 -0
  16. package/generators/lib/data/table-order.js +126 -0
  17. package/generators/lib/data/value-codec.js +188 -0
  18. package/generators/lib/templates/dashboard-catalog.js +1 -1
  19. package/generators/lib/templates/db-connection-env.js +1 -1
  20. package/generators/lib/templates/dbschema-catalog.js +1 -1
  21. package/generators/lib/templates/field-validation-catalog.js +1 -1
  22. package/generators/lib/templates/mysql-template.js +1 -1
  23. package/generators/lib/templates/oracle-template.js +1 -1
  24. package/generators/lib/templates/postgres-template.js +1 -1
  25. package/generators/lib/templates/query-declarative-catalog.js +1 -1
  26. package/generators/lib/templates/sqlite-template.js +1 -1
  27. package/integrity-manifest.json +18 -18
  28. package/package.json +1 -1
  29. package/scripts/verify-integrity.js +1 -1
  30. package/server.js +1 -1
  31. package/src/components/handlers/adjust_handler.js +1 -1
  32. package/src/components/handlers/audit_handler.js +1 -1
  33. package/src/components/handlers/delete_handler.js +1 -1
  34. package/src/components/handlers/export_handler.js +1 -1
  35. package/src/components/handlers/import_handler.js +1 -1
  36. package/src/components/handlers/insert_handler.js +1 -1
  37. package/src/components/handlers/update_handler.js +1 -1
  38. package/src/components/handlers/upload_handler.js +1 -1
  39. package/src/components/handlers/workflow_handler.js +1 -1
  40. package/src/components/integrations/webhook.js +1 -1
  41. package/src/consumers/baseConsumer.js +1 -1
  42. package/src/consumers/declarativeMapper.js +1 -1
  43. package/src/consumers/handlers/apiHandler.js +1 -1
  44. package/src/consumers/handlers/consoleHandler.js +1 -1
  45. package/src/consumers/handlers/databaseHandler.js +1 -1
  46. package/src/consumers/handlers/index.js +1 -1
  47. package/src/consumers/handlers/kafkaHandler.js +1 -1
  48. package/src/consumers/index.js +1 -1
  49. package/src/consumers/messageTransformer.js +1 -1
  50. package/src/consumers/validator.js +1 -1
  51. package/src/core/db/dialect/base-dialect.js +1 -1
  52. package/src/core/db/dialect/index.js +1 -1
  53. package/src/core/db/dialect/mysql-dialect.js +1 -1
  54. package/src/core/db/dialect/oracle-dialect.js +1 -1
  55. package/src/core/db/dialect/postgres-dialect.js +1 -1
  56. package/src/core/db/dialect/sqlite-dialect.js +1 -1
  57. package/src/core/db/flatten-helper.js +1 -1
  58. package/src/core/db/query-builder-error.js +1 -1
  59. package/src/core/db/query-builder.js +1 -1
  60. package/src/core/db/relation-helper.js +1 -1
  61. package/src/core/handlers/delete_handler.js +1 -1
  62. package/src/core/handlers/insert_handler.js +1 -1
  63. package/src/core/handlers/update_handler.js +1 -1
  64. package/src/core/models/base-model.js +1 -1
  65. package/src/core/utils/cache-manager.js +1 -1
  66. package/src/core/utils/component-engine.js +1 -1
  67. package/src/core/utils/context-builder.js +1 -1
  68. package/src/core/utils/datetime-formatter.js +1 -1
  69. package/src/core/utils/datetime-parser.js +1 -1
  70. package/src/core/utils/db.js +1 -1
  71. package/src/core/utils/logger.js +1 -1
  72. package/src/core/utils/payload-loader.js +1 -1
  73. package/src/core/utils/security-checks.js +1 -1
  74. package/src/middleware/body-options.js +1 -1
  75. package/src/middleware/cors.js +1 -1
  76. package/src/middleware/idempotency.js +1 -1
  77. package/src/middleware/rate-limiter.js +1 -1
  78. package/src/middleware/request-logger.js +1 -1
  79. package/src/middleware/security-headers.js +1 -1
  80. package/src/models/base-model-mysql.js +1 -1
  81. package/src/models/base-model-oracle.js +1 -1
  82. package/src/models/base-model-sqlite.js +1 -1
  83. package/src/models/base-model.js +1 -1
  84. package/src/pro/caching/redis-client.js +1 -1
  85. package/src/pro/caching/redis-helper.js +1 -1
  86. package/src/pro/consumers/baseConsumer.js +1 -1
  87. package/src/pro/consumers/declarativeMapper.js +1 -1
  88. package/src/pro/consumers/handlers/apiHandler.js +1 -1
  89. package/src/pro/consumers/handlers/consoleHandler.js +1 -1
  90. package/src/pro/consumers/handlers/databaseHandler.js +1 -1
  91. package/src/pro/consumers/handlers/index.js +1 -1
  92. package/src/pro/consumers/handlers/kafkaHandler.js +1 -1
  93. package/src/pro/consumers/index.js +1 -1
  94. package/src/pro/consumers/messageTransformer.js +1 -1
  95. package/src/pro/consumers/validator.js +1 -1
  96. package/src/pro/database/base-model-mysql.js +1 -1
  97. package/src/pro/database/base-model-oracle.js +1 -1
  98. package/src/pro/database/base-model-sqlite.js +1 -1
  99. package/src/pro/database/db-mysql.js +1 -1
  100. package/src/pro/database/db-oracle.js +1 -1
  101. package/src/pro/database/db-sqlite.js +1 -1
  102. package/src/pro/excel/excel-generator.js +1 -1
  103. package/src/pro/excel/excel-parser.js +1 -1
  104. package/src/pro/excel/export-service.js +1 -1
  105. package/src/pro/excel/export_handler.js +1 -1
  106. package/src/pro/excel/import-service.js +1 -1
  107. package/src/pro/excel/import-validator.js +1 -1
  108. package/src/pro/excel/import_handler.js +1 -1
  109. package/src/pro/excel/upsert-builder.js +1 -1
  110. package/src/pro/idgen/idgen-routes.js +1 -1
  111. package/src/pro/integrations/lookup-resolver.js +1 -1
  112. package/src/pro/integrations/upload-handler-v2.js +1 -1
  113. package/src/pro/integrations/upload-handler.js +1 -1
  114. package/src/pro/integrations/webhook.js +1 -1
  115. package/src/pro/locking/lock-routes.js +1 -1
  116. package/src/pro/locking/resource-lock-manager.js +1 -1
  117. package/src/pro/messaging/kafkaConsumerService.js +1 -1
  118. package/src/pro/messaging/kafkaService.js +1 -1
  119. package/src/pro/messaging/messagehubService.js +1 -1
  120. package/src/pro/messaging/rabbitmqService.js +1 -1
  121. package/src/pro/scheduler/job-manager.js +1 -1
  122. package/src/pro/scheduler/job-routes.js +1 -1
  123. package/src/pro/scheduler/job-validator.js +1 -1
  124. package/src/pro/storage/base-storage-provider.js +1 -1
  125. package/src/pro/storage/file-metadata-helper.js +1 -1
  126. package/src/pro/storage/index.js +1 -1
  127. package/src/pro/storage/local-storage-provider.js +1 -1
  128. package/src/pro/storage/s3-storage-provider.js +1 -1
  129. package/src/pro/storage/upload-cleanup-job.js +1 -1
  130. package/src/pro/storage/upload-cleanup-scheduler.js +1 -1
  131. package/src/pro/storage/upload-pending-tracker.js +1 -1
  132. package/src/pro/websocket/broadcast-helper.js +1 -1
  133. package/src/pro/websocket/index.js +1 -1
  134. package/src/pro/websocket/livesync-server.js +1 -1
  135. package/src/pro/websocket/ws-broadcaster.js +1 -1
  136. package/src/services/export-service.js +1 -1
  137. package/src/services/import-service.js +1 -1
  138. package/src/services/kafkaConsumerService.js +1 -1
  139. package/src/services/kafkaService.js +1 -1
  140. package/src/services/messagehubService.js +1 -1
  141. package/src/services/rabbitmqService.js +1 -1
  142. package/src/utils/cache-invalidation-registry.js +1 -1
  143. package/src/utils/cache-manager.js +1 -1
  144. package/src/utils/component-engine.js +1 -1
  145. package/src/utils/config-extractor.js +1 -1
  146. package/src/utils/consumerLogger.js +1 -1
  147. package/src/utils/context-builder.js +1 -1
  148. package/src/utils/dashboard-helpers.js +1 -1
  149. package/src/utils/dateHelper.js +1 -1
  150. package/src/utils/datetime-formatter.js +1 -1
  151. package/src/utils/datetime-parser.js +1 -1
  152. package/src/utils/db-bootstrap.js +1 -1
  153. package/src/utils/db-mysql.js +1 -1
  154. package/src/utils/db-oracle.js +1 -1
  155. package/src/utils/db-sqlite.js +1 -1
  156. package/src/utils/db.js +1 -1
  157. package/src/utils/demo-generator.js +1 -1
  158. package/src/utils/excel-generator.js +1 -1
  159. package/src/utils/excel-parser.js +1 -1
  160. package/src/utils/file-watcher.js +1 -1
  161. package/src/utils/id-generator.js +1 -1
  162. package/src/utils/idempotency-manager.js +1 -1
  163. package/src/utils/import-validator.js +1 -1
  164. package/src/utils/license-client.js +1 -1
  165. package/src/utils/lock-manager.js +1 -1
  166. package/src/utils/logger.js +1 -1
  167. package/src/utils/lookup-resolver.js +1 -1
  168. package/src/utils/payload-loader.js +1 -1
  169. package/src/utils/processor-response.js +1 -1
  170. package/src/utils/rabbitmq.js +1 -1
  171. package/src/utils/redis-client.js +1 -1
  172. package/src/utils/redis-helper.js +1 -1
  173. package/src/utils/request-scope.js +1 -1
  174. package/src/utils/security-checks.js +1 -1
  175. package/src/utils/service-resolver.js +1 -1
  176. package/src/utils/shutdown-coordinator.js +1 -1
  177. package/src/utils/trusted-keys.js +1 -1
  178. package/src/utils/upload-handler.js +1 -1
  179. package/src/utils/upsert-builder.js +1 -1
  180. package/src/utils/workflow-hook-executor.js +1 -1
@@ -0,0 +1,236 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * envelope — Bentuk + I/O envelope file data.
5
+ *
6
+ * Struktur envelope (urutan key wajib dipertahankan):
7
+ * { format_version, schema, table, source_dialect, columns, rows }
8
+ *
9
+ * `schema` (= `ir.schemaName`, `null` bila tanpa schema) ditambahkan di format
10
+ * version 1.1. File 1.0 (tanpa `schema`) tetap diterima oleh `validateEnvelope`
11
+ * untuk back-compat baca `data push`.
12
+ *
13
+ * Mengacu pada docs/plan/data-pull-push/data-pull-push-03-data-format.md.
14
+ *
15
+ * @module generators/lib/data/envelope
16
+ */
17
+
18
+ const FORMAT_VERSION = '1.1';
19
+
20
+ /**
21
+ * Bangun header envelope (tanpa rows).
22
+ *
23
+ * @param {Object} args
24
+ * @param {string|null} [args.schema] - Nama schema (= ir.schemaName; null bila tanpa schema)
25
+ * @param {string} args.table - Nama tabel
26
+ * @param {string} args.sourceDialect - Dialect sumber (mis. 'postgresql')
27
+ * @param {Array<Object>} args.columns - Daftar kolom { name, type, nullable, primary_key }
28
+ * @returns {{format_version: string, schema: (string|null), table: string, source_dialect: string, columns: Array}}
29
+ */
30
+ function buildHeader({ schema = null, table, sourceDialect, columns }) {
31
+ return {
32
+ format_version: FORMAT_VERSION,
33
+ schema,
34
+ table,
35
+ source_dialect: sourceDialect,
36
+ columns
37
+ };
38
+ }
39
+
40
+ /**
41
+ * Resolve sink penulisan menjadi fungsi write(chunk).
42
+ * Menerima: fungsi, object dengan method `write`, atau undefined (buffer mode).
43
+ *
44
+ * @param {Function|{write: Function}|undefined} sink
45
+ * @returns {{write: Function, buffer: (string[]|null)}}
46
+ */
47
+ function resolveSink(sink) {
48
+ if (sink === undefined || sink === null) {
49
+ const buffer = [];
50
+ return { write: (chunk) => buffer.push(chunk), buffer };
51
+ }
52
+ if (typeof sink === 'function') {
53
+ return { write: sink, buffer: null };
54
+ }
55
+ if (typeof sink.write === 'function') {
56
+ return { write: (chunk) => sink.write(chunk), buffer: null };
57
+ }
58
+ throw new Error('EnvelopeWriter: sink harus berupa fungsi, object dengan method write(), atau undefined');
59
+ }
60
+
61
+ /**
62
+ * Penulis envelope streaming.
63
+ *
64
+ * Menulis header, meng-append rows per batch, lalu menutup array `rows` dan envelope.
65
+ * Cocok untuk tabel besar karena rows tidak perlu ditahan seluruhnya di memori.
66
+ * Output akhir adalah JSON valid dengan urutan key:
67
+ * format_version, schema, table, source_dialect, columns, rows
68
+ *
69
+ * Penggunaan:
70
+ * const w = new EnvelopeWriter(fs.createWriteStream(path));
71
+ * w.writeHeader({ schema, table, sourceDialect, columns });
72
+ * w.appendRows(batch1);
73
+ * w.appendRows(batch2);
74
+ * w.end();
75
+ *
76
+ * Tanpa sink (buffer mode), `end()` mengembalikan string JSON penuh.
77
+ */
78
+ class EnvelopeWriter {
79
+ /**
80
+ * @param {Function|{write: Function}} [sink] - Tujuan tulis; kosong = buffer mode
81
+ */
82
+ constructor(sink) {
83
+ const resolved = resolveSink(sink);
84
+ this._write = resolved.write;
85
+ this._buffer = resolved.buffer;
86
+ this._state = 'init'; // init → header → ended
87
+ this._rowsWritten = 0;
88
+ }
89
+
90
+ /**
91
+ * Tulis prefix header dan buka array `rows`.
92
+ * @param {Object} header - { schema, table, sourceDialect, columns }
93
+ * @returns {EnvelopeWriter} this
94
+ */
95
+ writeHeader({ schema, table, sourceDialect, columns }) {
96
+ if (this._state !== 'init') {
97
+ throw new Error('EnvelopeWriter: writeHeader() hanya boleh dipanggil sekali di awal');
98
+ }
99
+ const prefix =
100
+ '{' +
101
+ `"format_version":${JSON.stringify(FORMAT_VERSION)},` +
102
+ `"schema":${JSON.stringify(schema ?? null)},` +
103
+ `"table":${JSON.stringify(table)},` +
104
+ `"source_dialect":${JSON.stringify(sourceDialect)},` +
105
+ `"columns":${JSON.stringify(columns)},` +
106
+ '"rows":[';
107
+ this._write(prefix);
108
+ this._state = 'header';
109
+ return this;
110
+ }
111
+
112
+ /**
113
+ * Append satu batch rows ke array `rows`.
114
+ * @param {Array<Object>} rows
115
+ * @returns {EnvelopeWriter} this
116
+ */
117
+ appendRows(rows) {
118
+ if (this._state !== 'header') {
119
+ throw new Error('EnvelopeWriter: appendRows() harus dipanggil setelah writeHeader() dan sebelum end()');
120
+ }
121
+ if (!Array.isArray(rows)) {
122
+ throw new Error('EnvelopeWriter: appendRows() membutuhkan array rows');
123
+ }
124
+ for (const row of rows) {
125
+ const sep = this._rowsWritten === 0 ? '' : ',';
126
+ this._write(sep + JSON.stringify(row));
127
+ this._rowsWritten++;
128
+ }
129
+ return this;
130
+ }
131
+
132
+ /**
133
+ * Tutup array `rows` dan envelope.
134
+ * @returns {string|undefined} String JSON penuh bila buffer mode, selain itu undefined
135
+ */
136
+ end() {
137
+ if (this._state === 'init') {
138
+ throw new Error('EnvelopeWriter: end() dipanggil sebelum writeHeader()');
139
+ }
140
+ if (this._state === 'ended') {
141
+ throw new Error('EnvelopeWriter: end() sudah dipanggil');
142
+ }
143
+ this._write(']}');
144
+ this._state = 'ended';
145
+ if (this._buffer) {
146
+ return this._buffer.join('');
147
+ }
148
+ return undefined;
149
+ }
150
+ }
151
+
152
+ /**
153
+ * Parse string JSON envelope menjadi object.
154
+ *
155
+ * @param {string} jsonString
156
+ * @returns {Object} Object envelope
157
+ * @throws {Error} exitCode=1 bila JSON tidak dapat di-parse
158
+ */
159
+ function parseEnvelope(jsonString) {
160
+ try {
161
+ return JSON.parse(jsonString);
162
+ } catch (err) {
163
+ const e = new Error(`Envelope tidak dapat di-parse sebagai JSON: ${err.message}`);
164
+ e.exitCode = 1;
165
+ throw e;
166
+ }
167
+ }
168
+
169
+ /**
170
+ * Validasi shape envelope. Throw (exitCode=1) bila tidak valid.
171
+ *
172
+ * Wajib: format_version (ada), table (string), columns (array of {name,type}),
173
+ * rows (array of object). Opsional: `schema` (string atau null bila ada).
174
+ *
175
+ * `format_version` tetap LENIENT (cukup ada/non-null) agar file 1.0 maupun 1.1
176
+ * sama-sama diterima tanpa penolakan versi. `schema` opsional untuk back-compat:
177
+ * file 1.0 (tanpa key `schema`) diterima; bila ada, wajib string atau null.
178
+ *
179
+ * @param {Object} obj
180
+ * @returns {Object} obj yang sama (untuk chaining)
181
+ * @throws {Error} exitCode=1 bila shape tidak valid
182
+ */
183
+ function validateEnvelope(obj) {
184
+ const fail = (msg) => {
185
+ const e = new Error(`Envelope tidak valid: ${msg}`);
186
+ e.exitCode = 1;
187
+ throw e;
188
+ };
189
+
190
+ if (obj === null || typeof obj !== 'object' || Array.isArray(obj)) {
191
+ fail('harus berupa object');
192
+ }
193
+ if (obj.format_version === undefined || obj.format_version === null) {
194
+ fail('field "format_version" wajib ada');
195
+ }
196
+ // `schema` opsional (back-compat 1.0). Bila absen → diterima; bila ada → wajib
197
+ // string atau null. `null` eksplisit (1.1 tabel schemaless) tetap valid.
198
+ if (obj.schema !== undefined && obj.schema !== null && typeof obj.schema !== 'string') {
199
+ fail('field "schema" wajib berupa string atau null bila ada');
200
+ }
201
+ if (typeof obj.table !== 'string' || obj.table === '') {
202
+ fail('field "table" wajib berupa string non-kosong');
203
+ }
204
+ if (!Array.isArray(obj.columns)) {
205
+ fail('field "columns" wajib berupa array');
206
+ }
207
+ obj.columns.forEach((col, i) => {
208
+ if (col === null || typeof col !== 'object' || Array.isArray(col)) {
209
+ fail(`columns[${i}] harus berupa object`);
210
+ }
211
+ if (typeof col.name !== 'string' || col.name === '') {
212
+ fail(`columns[${i}].name wajib berupa string non-kosong`);
213
+ }
214
+ if (typeof col.type !== 'string' || col.type === '') {
215
+ fail(`columns[${i}].type wajib berupa string non-kosong`);
216
+ }
217
+ });
218
+ if (!Array.isArray(obj.rows)) {
219
+ fail('field "rows" wajib berupa array');
220
+ }
221
+ obj.rows.forEach((row, i) => {
222
+ if (row === null || typeof row !== 'object' || Array.isArray(row)) {
223
+ fail(`rows[${i}] harus berupa object`);
224
+ }
225
+ });
226
+
227
+ return obj;
228
+ }
229
+
230
+ module.exports = {
231
+ FORMAT_VERSION,
232
+ buildHeader,
233
+ EnvelopeWriter,
234
+ parseEnvelope,
235
+ validateEnvelope
236
+ };
@@ -0,0 +1,414 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * pull-runner — Business logic command `data pull`.
5
+ *
6
+ * `run()` adalah dispatcher (single vs multi-schema); `runOne(ir, ctx)` mengeksekusi
7
+ * pull satu tabel dan mengelola executor-nya sendiri (Phase 02, data-pull-push-v2).
8
+ *
9
+ * Alur dispatcher `run()`:
10
+ * 1. Resolve config DB (config-resolver) → loadConfig (dbschema-kit/connection); SEKALI.
11
+ * 2. parseSchemaFlag + validateScopeFlags (sebelum load SDF / koneksi DB): tepat satu
12
+ * dari --table / --schema / --all-schemas (selain itu → exit 2).
13
+ * 3. Validasi --format (hanya 'json').
14
+ * 4. Load SDF SEKALI → Map<qualifiedName, IR> (sdf-reader).
15
+ * 5. resolveScope(models, args, schemaList) → daftar IR sesuai flag scope.
16
+ * 5a. Jalur --table: satu IR → runOne(ir, ctx) → ringkasan single.
17
+ * 5b. Jalur --schema/--all-schemas: iterasi subset FAIL-FAST (Q3) → runOne per IR →
18
+ * ringkasan agregat. Tabel yang sudah selesai sebelum error tetap tertulis.
19
+ *
20
+ * Alur `runOne(ir, ctx)` (per tabel; metadata kolom/namespace/PK murni dari SDF IR):
21
+ * a. Cek file output; tolak bila sudah ada tanpa --force (sebelum konek/tulis).
22
+ * b. Validasi identifier tabel/kolom (anti-injection) + no-PK guard.
23
+ * c. Connect (db-executor) + SELECT bertahap (keyset PK tunggal, OFFSET fallback composite).
24
+ * d. Tulis envelope streaming (EnvelopeWriter) + encode nilai per kolom (value-codec).
25
+ * e. Disconnect (selalu) + cleanup file partial saat error.
26
+ *
27
+ * Exit code: 0 sukses; 1 operational (file exists, baca SDF gagal); 2 usage/precondition
28
+ * (--config wajib, mutual-exclusive, tabel tak terdaftar, --format invalid); 3 database
29
+ * error. Handler tidak memanggil process.exit() sendiri.
30
+ *
31
+ * @module generators/lib/data/pull-runner
32
+ */
33
+
34
+ const fs = require('fs');
35
+ const path = require('path');
36
+
37
+ const { resolveConfig, printDefaultConfigWarning } = require('../utils/config-resolver');
38
+ const { loadConfig } = require('../dbschema-kit/connection');
39
+ const {
40
+ loadSdf,
41
+ irToColumns,
42
+ getNamespace,
43
+ getPrimaryKey
44
+ } = require('./sdf-reader');
45
+ const { parseSchemaFlag, validateScopeFlags, resolveScope, relDataFilePath } = require('./data-scope');
46
+ const { createDialect, validateIdentifier } = require('./dialect-kit');
47
+ const { createExecutor } = require('./db-executor');
48
+ const { encodeValue } = require('./value-codec');
49
+ const { EnvelopeWriter } = require('./envelope');
50
+ const { QueryBuilder } = require('../../../src/core/db/query-builder');
51
+
52
+ const DEFAULT_BATCH_SIZE = 1000;
53
+
54
+ /**
55
+ * Buat Error dengan exitCode terlampir (override default dispatcher).
56
+ * @param {string} message
57
+ * @param {number} exitCode
58
+ * @returns {Error}
59
+ */
60
+ function failWith(message, exitCode) {
61
+ const err = new Error(message);
62
+ err.exitCode = exitCode;
63
+ return err;
64
+ }
65
+
66
+ /**
67
+ * Petakan `config.dialect` internal ke bentuk DB_TYPE kanonik untuk envelope.
68
+ *
69
+ * `loadConfig` memetakan `DB_TYPE=postgresql` menjadi `dialect='postgres'`; envelope
70
+ * (header `source_dialect`) dan round-trip `data push` (Phase 03) memakai bentuk
71
+ * kanonik `postgresql`. Dialect `mysql`/`oracle`/`sqlite` identik di kedua sisi.
72
+ *
73
+ * @param {string} dialect - config.dialect ('postgres'|'mysql'|'oracle'|'sqlite')
74
+ * @returns {string} 'postgresql'|'mysql'|'oracle'|'sqlite'
75
+ */
76
+ function canonicalSourceDialect(dialect) {
77
+ return dialect === 'postgres' ? 'postgresql' : dialect;
78
+ }
79
+
80
+ /**
81
+ * Normalisasi nilai integer non-negatif dari flag.
82
+ * @param {*} value
83
+ * @param {number|null} fallback
84
+ * @returns {number|null}
85
+ */
86
+ function normalizeInt(value, fallback) {
87
+ if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
88
+ return Math.trunc(value);
89
+ }
90
+ return fallback;
91
+ }
92
+
93
+ /**
94
+ * Encode satu row mentah menjadi representasi envelope, hanya kolom whitelist SDF
95
+ * dalam urutan deklarasi field.
96
+ * @param {Object} row
97
+ * @param {Array<{name: string, type: string}>} columns
98
+ * @returns {Object}
99
+ */
100
+ function encodeRow(row, columns) {
101
+ const out = {};
102
+ for (const col of columns) {
103
+ out[col.name] = encodeValue(row[col.name], col.type);
104
+ }
105
+ return out;
106
+ }
107
+
108
+ /**
109
+ * Baca rows tabel secara bertahap dan kirim tiap batch (sudah ter-encode) ke onBatch.
110
+ *
111
+ * - Keyset bila PK tunggal: ORDER BY pk, tiap batch WHERE pk > last LIMIT n.
112
+ * - OFFSET fallback bila PK composite: ORDER BY pk LIMIT n OFFSET m.
113
+ * - `limit` membatasi total rows; berhenti saat tercapai.
114
+ *
115
+ * @param {Object} opts
116
+ * @param {Object} opts.executor - db-executor (dbUtils untuk QueryBuilder)
117
+ * @param {Object} opts.dialect - instance dialect
118
+ * @param {string} opts.queryTable - nama tabel target (qualified bila ber-schema)
119
+ * @param {Array<{name: string, type: string}>} opts.columns - kolom whitelist SDF
120
+ * @param {string[]} opts.pk - daftar kolom primary key
121
+ * @param {number} opts.batchSize - ukuran batch
122
+ * @param {number|null} opts.limit - batas total rows
123
+ * @param {Function} opts.onBatch - callback(encodedRows) per batch
124
+ * @returns {Promise<number>} total rows yang di-pull
125
+ */
126
+ async function pullInBatches({ executor, dialect, queryTable, columns, pk, batchSize, limit, onBatch }) {
127
+ const columnNames = columns.map((c) => c.name);
128
+ const useKeyset = pk.length === 1;
129
+ // `run()` menjamin PK non-kosong (no-PK = hard error sebelum mencapai sini),
130
+ // sehingga ORDER BY selalu memakai kolom PK; tak ada fallback seluruh-kolom.
131
+ const orderCols = pk;
132
+
133
+ let total = 0;
134
+ let lastKey = null;
135
+ let offset = 0;
136
+
137
+ for (;;) {
138
+ if (limit !== null && total >= limit) break;
139
+
140
+ let perBatch = batchSize;
141
+ if (limit !== null) {
142
+ perBatch = Math.min(batchSize, limit - total);
143
+ }
144
+ if (perBatch <= 0) break;
145
+
146
+ const qb = new QueryBuilder(executor, dialect, queryTable);
147
+ qb.select(...columnNames);
148
+
149
+ if (useKeyset) {
150
+ if (lastKey !== null && lastKey !== undefined) {
151
+ qb.where(pk[0], '>', lastKey);
152
+ }
153
+ qb.orderBy(pk[0], 'asc');
154
+ qb.limit(perBatch);
155
+ } else {
156
+ for (const col of orderCols) qb.orderBy(col, 'asc');
157
+ qb.limit(perBatch);
158
+ qb.offset(offset);
159
+ }
160
+
161
+ const rows = await qb.get();
162
+ if (!rows || rows.length === 0) break;
163
+
164
+ if (useKeyset) {
165
+ lastKey = rows[rows.length - 1][pk[0]];
166
+ } else {
167
+ offset += rows.length;
168
+ }
169
+
170
+ onBatch(rows.map((row) => encodeRow(row, columns)));
171
+ total += rows.length;
172
+
173
+ if (rows.length < perBatch) break;
174
+ }
175
+
176
+ return total;
177
+ }
178
+
179
+ /**
180
+ * Pull satu tabel SDF ke file envelope JSON (path via relDataFilePath: bersarang
181
+ * `<output>/<schema>/<table>.json` bila ber-schema, flat `<output>/<table>.json`
182
+ * bila tanpa schema).
183
+ *
184
+ * Mengelola executor-nya sendiri (buat → disconnect, termasuk cleanup file partial
185
+ * saat error) agar isolasi per-tabel jelas dan jalur multi-schema dapat memanggilnya
186
+ * berulang tanpa kebocoran koneksi. Metadata kolom, namespace, dan PK murni dari IR.
187
+ *
188
+ * @param {Object} ir - IR model SDF satu tabel
189
+ * @param {Object} ctx - konteks resolusi sekali-pakai
190
+ * @param {Object} ctx.config - config DB hasil loadConfig
191
+ * @param {string} ctx.sourceDialect - dialect kanonik untuk header envelope
192
+ * @param {Object} ctx.args - argumen ter-parse (storage-path, force, limit, batch-size)
193
+ * @param {string} ctx.cwd - working directory
194
+ * @returns {Promise<{schema: (string|null), table: string, rows: number, file: string, source_dialect: string}>}
195
+ */
196
+ async function runOne(ir, ctx) {
197
+ const { config, sourceDialect, args, cwd } = ctx;
198
+
199
+ const columns = irToColumns(ir);
200
+ const { schemaName, qualifiedName } = getNamespace(ir);
201
+ const rawPk = getPrimaryKey(ir);
202
+ const pk = Array.isArray(rawPk) ? rawPk : [];
203
+
204
+ // ── a. Cek file output (SEBELUM menulis/konek) ───────────────────────────────
205
+ // Path diturunkan dari relDataFilePath (aturan seragam Q2): tabel ber-schema →
206
+ // bersarang `<schema>/<table>.json` (mis. `sales/order_item.json`); tabel tanpa
207
+ // schema → flat `<table>.json` (mis. `visitors.json`). Helper bersama dengan push
208
+ // (data-scope) agar derivasi tidak drift. queryTable SELECT tetap memakai
209
+ // qualifiedName (identitas DB).
210
+ const outputDir = path.resolve(cwd, args['storage-path']);
211
+ const outFile = path.join(outputDir, relDataFilePath(ir));
212
+ if (fs.existsSync(outFile) && args.force !== true) {
213
+ const rel = path.relative(cwd, outFile).split(path.sep).join('/');
214
+ throw failWith(
215
+ `Output file already exists: ${rel}. Use --force to overwrite.`,
216
+ 1
217
+ );
218
+ }
219
+
220
+ // ── b. Validasi identifier (anti-injection) + no-PK guard ────────────────────
221
+ const dialect = createDialect(config.dialect);
222
+ validateIdentifier(dialect, qualifiedName);
223
+ for (const col of columns) validateIdentifier(dialect, col.name);
224
+
225
+ // No-PK = anomali: schema-validator MEWAJIBKAN primary key, jadi IR yang lolos load
226
+ // SDF pasti punya PK. Bila tetap kosong (mis. IR diinjeksi), gagal jelas lebih aman
227
+ // daripada melanjutkan dengan urutan baris tak tentu.
228
+ if (pk.length === 0) {
229
+ throw failWith(`Table '${ir.tableName}' has no primary key in SDF (unexpected)`, 1);
230
+ }
231
+
232
+ const executor = createExecutor(config);
233
+
234
+ const batchSize = normalizeInt(args['batch-size'], DEFAULT_BATCH_SIZE) || DEFAULT_BATCH_SIZE;
235
+ const limit = normalizeInt(args.limit, null);
236
+
237
+ // ── c-d. SELECT bertahap + tulis envelope streaming (sinkron) ─────────────────
238
+ // Penulisan via fs.writeSync (bukan WriteStream) menjaga streaming — tiap chunk
239
+ // langsung ditulis ke OS tanpa menahan seluruh envelope di memory — sekaligus
240
+ // sinkron sehingga satu-satunya async di runOne() adalah query DB.
241
+ let rowCount = 0;
242
+ let fd = null;
243
+ try {
244
+ // Buat direktori parent dari outFile (bukan sekadar outputDir): pada layout
245
+ // bersarang, outFile berada di subfolder schema (mis. `<storage>/main/`) yang
246
+ // belum tentu ada. path.dirname mencakup subfolder schema maupun flat root.
247
+ fs.mkdirSync(path.dirname(outFile), { recursive: true });
248
+ fd = fs.openSync(outFile, 'w');
249
+
250
+ const writer = new EnvelopeWriter((chunk) => fs.writeSync(fd, chunk));
251
+ writer.writeHeader({ schema: schemaName, table: ir.tableName, sourceDialect, columns });
252
+
253
+ rowCount = await pullInBatches({
254
+ executor,
255
+ dialect,
256
+ queryTable: qualifiedName,
257
+ columns,
258
+ pk,
259
+ batchSize,
260
+ limit,
261
+ onBatch: (encodedRows) => writer.appendRows(encodedRows)
262
+ });
263
+
264
+ writer.end();
265
+ fs.closeSync(fd);
266
+ fd = null;
267
+ } catch (e) {
268
+ // Tutup fd SEBELUM unlink (Windows menolak unlink file yang masih terbuka).
269
+ if (fd !== null) {
270
+ try { fs.closeSync(fd); } catch (_e) { /* ignore */ }
271
+ fd = null;
272
+ }
273
+ // Bersihkan file partial agar tidak meninggalkan envelope tak lengkap.
274
+ try { fs.unlinkSync(outFile); } catch (_e) { /* sudah tidak ada */ }
275
+ // exitCode yang sudah di-set (mis. dari QueryBuilder/validasi) dipertahankan;
276
+ // selain itu error lapisan DB (connect/select) dipetakan ke exit 3.
277
+ if (Number.isInteger(e.exitCode)) throw e;
278
+ throw failWith(`Database error during pull: ${e.message}`, 3);
279
+ } finally {
280
+ try { await executor.disconnect(); } catch (_e) { /* ignore */ }
281
+ }
282
+
283
+ // ── e. Ringkasan per-tabel (tanpa `command`; dirakit caller) ─────────────────
284
+ const relFile = path.relative(cwd, outFile).split(path.sep).join('/');
285
+ return {
286
+ schema: schemaName,
287
+ table: ir.tableName,
288
+ rows: rowCount,
289
+ file: relFile,
290
+ source_dialect: sourceDialect
291
+ };
292
+ }
293
+
294
+ /**
295
+ * Jalankan command `data pull` (dispatcher: single-table vs multi-schema).
296
+ * @param {Object} args - argumen ter-parse dari contract
297
+ * @returns {Promise<Object>} ringkasan single { command, schema, table, rows, file, source_dialect }
298
+ * atau agregat { command, scope, table_count, total_rows, tables: [...] }
299
+ */
300
+ async function run(args) {
301
+ const cwd = process.cwd();
302
+
303
+ // ── 1. Resolve config DB (SEKALI) ────────────────────────────────────────────
304
+ const resolved = resolveConfig(args.config, cwd);
305
+ if (!resolved) {
306
+ process.stderr.write(
307
+ "Tip: set a default with 'npx restforge config set-default --config=<file>' to omit --config in future runs\n"
308
+ );
309
+ throw failWith('--config=<file> is required.', 2);
310
+ }
311
+ if (resolved.source === 'default') {
312
+ printDefaultConfigWarning(resolved.defaultName);
313
+ }
314
+
315
+ let config;
316
+ try {
317
+ config = loadConfig(resolved.path);
318
+ } catch (e) {
319
+ throw failWith(e.message, 2);
320
+ }
321
+
322
+ // ── 2. Validasi scope flags (SEBELUM SDF/DB) ─────────────────────────────────
323
+ // Tepat satu dari --table / --schema / --all-schemas (selain itu → exit 2).
324
+ // Helper bersama dengan push (data-scope) agar perilaku identik.
325
+ const schemaList = parseSchemaFlag(args.schema);
326
+ validateScopeFlags(args, schemaList);
327
+ const hasTable = args.table != null && args.table !== '';
328
+
329
+ // ── 3. Validasi --format ─────────────────────────────────────────────────────
330
+ if (args.format !== 'json') {
331
+ throw failWith(
332
+ `Unsupported --format='${args.format}'. Only 'json' is supported in this version.`,
333
+ 2
334
+ );
335
+ }
336
+
337
+ // ── 4. Load SDF (SEKALI) ─────────────────────────────────────────────────────
338
+ const schemaPath = args['schema-path'];
339
+ const absSchemaPath = path.resolve(cwd, schemaPath);
340
+ let models;
341
+ try {
342
+ models = loadSdf(absSchemaPath);
343
+ } catch (e) {
344
+ // Error baca SDF (path tidak ada, parse gagal) = operational.
345
+ throw failWith(e.message, 1);
346
+ }
347
+
348
+ const ctx = {
349
+ config,
350
+ sourceDialect: canonicalSourceDialect(config.dialect),
351
+ args,
352
+ cwd
353
+ };
354
+
355
+ // ── 5. Resolusi scope → daftar IR ────────────────────────────────────────────
356
+ // --table → [ir] (exit 2 bila tak terdaftar/ambigu); --schema → subset (exit 2
357
+ // bila 0-match); --all-schemas → seluruh IR (boleh kosong → agregat 0 tabel).
358
+ const subset = resolveScope(models, args, schemaList);
359
+
360
+ // ── 5a. Jalur --table (single; regresi nol) ──────────────────────────────────
361
+ if (hasTable) {
362
+ const per = await runOne(subset[0], ctx);
363
+ const summary = { command: 'pull', ...per };
364
+
365
+ if (args.json === true) {
366
+ process.stdout.write(JSON.stringify(summary) + '\n');
367
+ } else {
368
+ process.stdout.write(
369
+ `Pulled ${per.rows} rows from ${per.table} -> ${per.file} (dialect: ${per.source_dialect})\n`
370
+ );
371
+ }
372
+ return summary;
373
+ }
374
+
375
+ // ── 5b. Jalur --schema/--all-schemas (fail-fast Q3; agregat) ─────────────────
376
+ // Iterasi mengikuti urutan insertion Map (file SDF ter-sort alfabetis). Error
377
+ // pada satu tabel menghentikan loop (fail-fast); tabel yang sudah selesai
378
+ // sebelum error tetap tertulis (konsekuensi Q3/R4, didokumentasikan Phase 03).
379
+ const scope = schemaList ? 'schema' : 'all-schemas';
380
+ const tables = [];
381
+ let totalRows = 0;
382
+ for (const ir of subset) {
383
+ const per = await runOne(ir, ctx);
384
+ tables.push(per);
385
+ totalRows += per.rows;
386
+ if (args.json !== true) {
387
+ process.stdout.write(
388
+ `Pulled ${per.rows} rows from ${per.table} -> ${per.file} (dialect: ${per.source_dialect})\n`
389
+ );
390
+ }
391
+ }
392
+
393
+ const aggregate = {
394
+ command: 'pull',
395
+ scope,
396
+ table_count: tables.length,
397
+ total_rows: totalRows,
398
+ tables
399
+ };
400
+
401
+ if (args.json === true) {
402
+ process.stdout.write(JSON.stringify(aggregate) + '\n');
403
+ } else {
404
+ process.stdout.write(`Done: ${tables.length} tables, ${totalRows} rows total\n`);
405
+ }
406
+
407
+ return aggregate;
408
+ }
409
+
410
+ module.exports = {
411
+ run,
412
+ // Diekspor untuk reuse/test internal.
413
+ _internal: { runOne, pullInBatches, encodeRow, normalizeInt, canonicalSourceDialect }
414
+ };