@rex0220/ksql-flownet 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.ja.md +81 -0
- package/README.md +117 -0
- package/dist/bundle/bundle-builder.d.ts +49 -0
- package/dist/bundle/bundle-builder.js +208 -0
- package/dist/bundle/index.d.ts +4 -0
- package/dist/bundle/index.js +4 -0
- package/dist/bundle/upload.d.ts +28 -0
- package/dist/bundle/upload.js +72 -0
- package/dist/bundle/zip-store.d.ts +17 -0
- package/dist/bundle/zip-store.js +175 -0
- package/dist/cli/archive-run-command.d.ts +12 -0
- package/dist/cli/archive-run-command.js +131 -0
- package/dist/cli/cancel-run-command.d.ts +9 -0
- package/dist/cli/cancel-run-command.js +77 -0
- package/dist/cli/force-unlock-network-command.d.ts +8 -0
- package/dist/cli/force-unlock-network-command.js +129 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.js +132 -0
- package/dist/cli/plan-command.d.ts +2 -0
- package/dist/cli/plan-command.js +100 -0
- package/dist/cli/poll-requests-command.d.ts +3 -0
- package/dist/cli/poll-requests-command.js +74 -0
- package/dist/cli/record-job-unlock-command.d.ts +11 -0
- package/dist/cli/record-job-unlock-command.js +78 -0
- package/dist/cli/resolve-node-command.d.ts +13 -0
- package/dist/cli/resolve-node-command.js +174 -0
- package/dist/cli/run-network-command.d.ts +14 -0
- package/dist/cli/run-network-command.js +418 -0
- package/dist/cli/status-command.d.ts +5 -0
- package/dist/cli/status-command.js +137 -0
- package/dist/cli/validate-command.d.ts +6 -0
- package/dist/cli/validate-command.js +17 -0
- package/dist/dag/descendants.d.ts +9 -0
- package/dist/dag/descendants.js +67 -0
- package/dist/dag/topological-sort.d.ts +7 -0
- package/dist/dag/topological-sort.js +32 -0
- package/dist/domain/business-key.d.ts +15 -0
- package/dist/domain/business-key.js +249 -0
- package/dist/domain/canonical-lock-key.d.ts +17 -0
- package/dist/domain/canonical-lock-key.js +60 -0
- package/dist/domain/canonical-record-key.d.ts +18 -0
- package/dist/domain/canonical-record-key.js +70 -0
- package/dist/domain/load-network.d.ts +4 -0
- package/dist/domain/load-network.js +47 -0
- package/dist/domain/network-definition.d.ts +47 -0
- package/dist/domain/network-definition.js +7 -0
- package/dist/domain/persistence-model.d.ts +198 -0
- package/dist/domain/persistence-model.js +2 -0
- package/dist/domain/run-aggregate.d.ts +3 -0
- package/dist/domain/run-aggregate.js +15 -0
- package/dist/domain/validate-network-path.d.ts +11 -0
- package/dist/domain/validate-network-path.js +33 -0
- package/dist/domain/validate-network.d.ts +3 -0
- package/dist/domain/validate-network.js +180 -0
- package/dist/executor/attempt-executor.d.ts +43 -0
- package/dist/executor/attempt-executor.js +236 -0
- package/dist/executor/job-log-reader.d.ts +37 -0
- package/dist/executor/job-log-reader.js +91 -0
- package/dist/executor/ksql-flow-cli.d.ts +86 -0
- package/dist/executor/ksql-flow-cli.js +161 -0
- package/dist/executor/preflight.d.ts +41 -0
- package/dist/executor/preflight.js +150 -0
- package/dist/executor/result-classifier.d.ts +69 -0
- package/dist/executor/result-classifier.js +237 -0
- package/dist/executor/run-subprocess.d.ts +64 -0
- package/dist/executor/run-subprocess.js +175 -0
- package/dist/io/input-baseline.d.ts +19 -0
- package/dist/io/input-baseline.js +214 -0
- package/dist/io/io-config.d.ts +6 -0
- package/dist/io/io-config.js +33 -0
- package/dist/io/io-path.d.ts +37 -0
- package/dist/io/io-path.js +279 -0
- package/dist/orchestration/archive-run.d.ts +53 -0
- package/dist/orchestration/archive-run.js +323 -0
- package/dist/orchestration/cancel-request.d.ts +18 -0
- package/dist/orchestration/cancel-request.js +67 -0
- package/dist/orchestration/ensure-run.d.ts +83 -0
- package/dist/orchestration/ensure-run.js +632 -0
- package/dist/orchestration/job-lock-audit.d.ts +24 -0
- package/dist/orchestration/job-lock-audit.js +123 -0
- package/dist/orchestration/orphan-attempt-adjudication.d.ts +39 -0
- package/dist/orchestration/orphan-attempt-adjudication.js +167 -0
- package/dist/orchestration/reconciliation.d.ts +41 -0
- package/dist/orchestration/reconciliation.js +526 -0
- package/dist/orchestration/resolve-node.d.ts +29 -0
- package/dist/orchestration/resolve-node.js +105 -0
- package/dist/orchestration/run-activity.d.ts +13 -0
- package/dist/orchestration/run-activity.js +14 -0
- package/dist/orchestration/sequential-scheduler.d.ts +79 -0
- package/dist/orchestration/sequential-scheduler.js +706 -0
- package/dist/orchestration/status.d.ts +91 -0
- package/dist/orchestration/status.js +171 -0
- package/dist/persistence/in-memory-repository.d.ts +41 -0
- package/dist/persistence/in-memory-repository.js +333 -0
- package/dist/persistence/kintone/client.d.ts +36 -0
- package/dist/persistence/kintone/client.js +93 -0
- package/dist/persistence/kintone/design-notes.d.ts +38 -0
- package/dist/persistence/kintone/design-notes.js +38 -0
- package/dist/persistence/kintone/repository.d.ts +44 -0
- package/dist/persistence/kintone/repository.js +836 -0
- package/dist/persistence/kintone/schema.d.ts +116 -0
- package/dist/persistence/kintone/schema.js +115 -0
- package/dist/persistence/network-lock-reader.d.ts +24 -0
- package/dist/persistence/network-lock-reader.js +43 -0
- package/dist/persistence/network-lock-recovery.d.ts +66 -0
- package/dist/persistence/network-lock-recovery.js +392 -0
- package/dist/persistence/network-lock.d.ts +97 -0
- package/dist/persistence/network-lock.js +408 -0
- package/dist/persistence/repository.d.ts +101 -0
- package/dist/persistence/repository.js +11 -0
- package/dist/persistence/state-transition.d.ts +4 -0
- package/dist/persistence/state-transition.js +25 -0
- package/dist/requests/flownet-child-client.d.ts +78 -0
- package/dist/requests/flownet-child-client.js +243 -0
- package/dist/requests/kintone-request-store.d.ts +44 -0
- package/dist/requests/kintone-request-store.js +202 -0
- package/dist/requests/poll-requests-config.d.ts +26 -0
- package/dist/requests/poll-requests-config.js +105 -0
- package/dist/requests/request-model.d.ts +57 -0
- package/dist/requests/request-model.js +305 -0
- package/dist/requests/request-poller.d.ts +33 -0
- package/dist/requests/request-poller.js +353 -0
- package/dist/requests/request-result.d.ts +18 -0
- package/dist/requests/request-result.js +215 -0
- package/dist/requests/start-request.d.ts +26 -0
- package/dist/requests/start-request.js +83 -0
- package/package.json +63 -0
- package/schemas/.gitkeep +1 -0
- package/schemas/network-definition.schema.json +121 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 rex0220
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.ja.md
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# kSQL-FlowNet
|
|
2
|
+
|
|
3
|
+
[English](./README.md) | 日本語
|
|
4
|
+
|
|
5
|
+
kSQL-FlowNetは、複数の[kSQL-Flow](https://www.npmjs.com/package/@rex0220/ksql-flow)ジョブをnetwork(DAG)として管理するControl Plane CLIです。network定義の検証、業務キー単位のRun一意性、依存関係に基づく直列実行、再開(resume)、Networkロック、状態保存、監査を担当します。分岐・合流を持つDAGを定義できますが、ノードは安定したトポロジカル順で1件ずつ直列実行します。
|
|
6
|
+
|
|
7
|
+
kintone上の「Run状況」ボードプラグインと操作要求アプリを組み合わせることで、リラン・停止・解除・新規実行(START)を画面から指示できます。
|
|
8
|
+
|
|
9
|
+
- **導入手順書**: [docs/installation.md](./docs/installation.md)(kintoneアプリ・トークン・プラグイン・サーバー・cronを0から本番運用まで)
|
|
10
|
+
- **仕様・運用文書**: [docs/README.md](./docs/README.md)(統合仕様書・一次対応1ページ・復旧runbook)
|
|
11
|
+
- **kintoneアプリの作成**: [templates/README.md](./templates/README.md)
|
|
12
|
+
- **ボードプラグイン**: [plugin/README.md](./plugin/README.md)
|
|
13
|
+
|
|
14
|
+
## 動作環境
|
|
15
|
+
|
|
16
|
+
- kintone(APIトークン・プラグイン・関連レコード・アプリテンプレートを使用)
|
|
17
|
+
- 実行サーバー: Node.js 22以上。通信は実行サーバーからkintoneへのHTTPS発信のみで、kintoneから実行サーバーへの接続はない(受信ポート・固定IP・ドメイン不要)
|
|
18
|
+
- 詳細は[統合仕様書 §2 動作環境](./docs/specification.md)を参照
|
|
19
|
+
|
|
20
|
+
## インストール
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
npm install --global @rex0220/ksql-flownet
|
|
24
|
+
ksql-flownet --version
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
kintoneアプリ(同梱テンプレート)・APIトークン・ボードプラグイン・サーバー環境・cronまでの導入手順は[docs/installation.md](./docs/installation.md)にあります。
|
|
28
|
+
|
|
29
|
+
## 開発
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
npm install
|
|
33
|
+
npm run build
|
|
34
|
+
npm run format:check
|
|
35
|
+
npm run lint
|
|
36
|
+
npm run typecheck
|
|
37
|
+
npm test
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## CLIの使い方
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
ksql-flownet --help
|
|
44
|
+
ksql-flownet --version
|
|
45
|
+
ksql-flownet validate path/to/network.yaml
|
|
46
|
+
ksql-flownet poll-requests --check
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
`validate`は、YAMLスキーマ・DAG規則・参照SQLファイルを、外部状態を変更せずに検証します。
|
|
50
|
+
|
|
51
|
+
`poll-requests`はkintone操作要求アプリのワンショットポーラーです。`REQUESTED`レコードをclaimし、`RERUN`・`STOP`・`RELEASE`・`START`を実行します。cron等のスケジューラから定期起動してください。設定は[`.env.example`](./.env.example)の`KSQL_FLOWNET_REQUEST_*`と、絶対パスのallowlistで行います:
|
|
52
|
+
|
|
53
|
+
```yaml
|
|
54
|
+
networks:
|
|
55
|
+
- network_id: monthly_jobs
|
|
56
|
+
definition_path: C:/srv/my-ksql-jobs/networks/monthly.yaml
|
|
57
|
+
app_start: false
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`app_start`はfail-closedです。省略は`false`と同じで、明示的なboolean `true`を設定したnetworkだけがSTART要求(アプリからの新規Run起動)の対象になります。このフラグは`RERUN`・`STOP`・`RELEASE`のRun検索対象からnetworkを外しません。
|
|
61
|
+
|
|
62
|
+
本番スケジュールを有効化する前に`poll-requests --check`を実行してください。これは読み取り専用の事前検査で、allowlistの全network定義と`network_id`を検証し、要求アプリへのGETアクセスを確認します。要求のclaim・更新や子プロセス(`status`・`run-network`・`cancel-run`)の起動は行いません。終了コードが0以外の場合はスケジュールを有効化しないでください。
|
|
63
|
+
|
|
64
|
+
各ノードは`KSQL_FLOWNET_PROFILE + ":" + nodes[].job_id`が64 UTF-16単位以内である必要があります。これはkSQL-Flowのジョブロックキーの実測上限です。現行の`validate`は超過を検出せず、実行時に`VALIDATION_ERROR`で失敗します。
|
|
65
|
+
|
|
66
|
+
WindowsでkSQL-Flowのソースビルドに対して`run-network`を実行する場合は、実行ファイルと先頭のCLIスクリプト引数を分けて設定します。引数に空白が含まれる場合はJSON配列形式を使ってください:
|
|
67
|
+
|
|
68
|
+
```powershell
|
|
69
|
+
$env:KSQL_FLOW_BIN = 'node.exe'
|
|
70
|
+
$env:KSQL_FLOW_BIN_ARGS = '["C:\\path\\to\\ksql-flow\\dist\\cli.js"]'
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`KSQL_FLOW_BIN_ARGS`は空白区切りの引数も受け付けます。スタンドアロン実行ファイルを使う場合は、`KSQL_FLOW_BIN`へ実行ファイルを指定し、`KSQL_FLOW_BIN_ARGS`は未設定にできます。
|
|
74
|
+
|
|
75
|
+
## 実機E2E
|
|
76
|
+
|
|
77
|
+
[`tests/e2e/README.md`](./tests/e2e/README.md)に記載の実機環境を設定したうえで、各シナリオをPowerShellから直列に実行します。これらは設定済みのkintoneとkSQL-Flow環境へアクセスするため、CIでは実行しないでください。各シナリオは`tests/e2e/results/`へサニタイズ済みの結果JSONを書き出し、自分のスコープの状態を清掃します。
|
|
78
|
+
|
|
79
|
+
## ライセンス
|
|
80
|
+
|
|
81
|
+
MIT。[LICENSE](./LICENSE)を参照してください。
|
package/README.md
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# kSQL-FlowNet
|
|
2
|
+
|
|
3
|
+
English | [日本語](./README.ja.md)
|
|
4
|
+
|
|
5
|
+
kSQL-FlowNet is a Control Plane CLI that manages multiple
|
|
6
|
+
[kSQL-Flow](https://www.npmjs.com/package/@rex0220/ksql-flow) jobs as a network
|
|
7
|
+
(DAG). It handles network-definition validation, Run uniqueness per business
|
|
8
|
+
key, dependency-ordered serial execution, resume, network locks, state
|
|
9
|
+
persistence, and audit trails. Networks may declare branching and joining DAGs,
|
|
10
|
+
but nodes execute one at a time in a stable topological order.
|
|
11
|
+
|
|
12
|
+
Combined with the "Run status" board plugin and the operation-request app on
|
|
13
|
+
kintone, operators can trigger rerun, stop, release, and new-run (START)
|
|
14
|
+
requests from the UI.
|
|
15
|
+
|
|
16
|
+
- **Installation guide**: [docs/installation.md](./docs/installation.md)
|
|
17
|
+
(kintone apps, tokens, plugin, server, cron — from zero to production)
|
|
18
|
+
- **Specification and operations docs**: [docs/README.md](./docs/README.md)
|
|
19
|
+
(integrated specification, first-response one-pager, recovery runbook)
|
|
20
|
+
- **Creating the kintone apps**: [templates/README.md](./templates/README.md)
|
|
21
|
+
- **Board plugin**: [plugin/README.md](./plugin/README.md)
|
|
22
|
+
|
|
23
|
+
## Requirements
|
|
24
|
+
|
|
25
|
+
- kintone (uses API tokens, a plugin, related records, and app templates)
|
|
26
|
+
- An execution server with Node.js 22 or later. All traffic is outbound HTTPS
|
|
27
|
+
from the server to kintone; kintone never connects to the server (no inbound
|
|
28
|
+
port, static IP, or domain required)
|
|
29
|
+
- See [Specification §2 (environment)](./docs/specification.md) for details
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
npm install --global @rex0220/ksql-flownet
|
|
35
|
+
ksql-flownet --version
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The full setup (kintone apps from the bundled app template, API tokens,
|
|
39
|
+
the board plugin, server environment, cron) is in
|
|
40
|
+
[docs/installation.md](./docs/installation.md).
|
|
41
|
+
|
|
42
|
+
## Development
|
|
43
|
+
|
|
44
|
+
```sh
|
|
45
|
+
npm install
|
|
46
|
+
npm run build
|
|
47
|
+
npm run format:check
|
|
48
|
+
npm run lint
|
|
49
|
+
npm run typecheck
|
|
50
|
+
npm test
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## CLI usage
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
ksql-flownet --help
|
|
57
|
+
ksql-flownet --version
|
|
58
|
+
ksql-flownet validate path/to/network.yaml
|
|
59
|
+
ksql-flownet poll-requests --check
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`validate` checks the YAML schema, DAG rules, and referenced SQL files without
|
|
63
|
+
changing external state.
|
|
64
|
+
|
|
65
|
+
`poll-requests` is a one-shot poller for the kintone operation-request app. It
|
|
66
|
+
claims `REQUESTED` records and performs `RERUN`, `STOP`, `RELEASE`, or `START`;
|
|
67
|
+
start it periodically from a scheduler such as cron. Configure it through the
|
|
68
|
+
`KSQL_FLOWNET_REQUEST_*` entries in [`.env.example`](./.env.example) and an
|
|
69
|
+
absolute-path allowlist such as:
|
|
70
|
+
|
|
71
|
+
```yaml
|
|
72
|
+
networks:
|
|
73
|
+
- network_id: monthly_jobs
|
|
74
|
+
definition_path: C:/srv/my-ksql-jobs/networks/monthly.yaml
|
|
75
|
+
app_start: false
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`app_start` is fail-closed: omitting it is equivalent to `false`, and only an
|
|
79
|
+
explicit boolean `true` enables START requests (launching a new Run from the
|
|
80
|
+
app) for that network. This flag does not remove the network from run lookup
|
|
81
|
+
for `RERUN`, `STOP`, or `RELEASE`.
|
|
82
|
+
|
|
83
|
+
Before enabling a production schedule, run `poll-requests --check`. This is a
|
|
84
|
+
read-only preflight: it validates every allowlisted network definition and its
|
|
85
|
+
`network_id`, then confirms GET access to the request app. It does not claim or
|
|
86
|
+
update requests and does not start `status`, `run-network`, or `cancel-run`
|
|
87
|
+
children. A nonzero exit must block schedule activation.
|
|
88
|
+
|
|
89
|
+
Each network node must also satisfy
|
|
90
|
+
`KSQL_FLOWNET_PROFILE + ":" + nodes[].job_id` <= 64 UTF-16 code units. This is
|
|
91
|
+
the measured kSQL-Flow job-lock-key limit. The current `validate` command does
|
|
92
|
+
not detect an overrun; execution fails later with `VALIDATION_ERROR`.
|
|
93
|
+
|
|
94
|
+
To run `run-network` against a source build of kSQL-Flow on Windows, set the
|
|
95
|
+
executable and its leading CLI-script argument separately. Use the JSON array
|
|
96
|
+
form when an argument contains spaces:
|
|
97
|
+
|
|
98
|
+
```powershell
|
|
99
|
+
$env:KSQL_FLOW_BIN = 'node.exe'
|
|
100
|
+
$env:KSQL_FLOW_BIN_ARGS = '["C:\\path\\to\\ksql-flow\\dist\\cli.js"]'
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`KSQL_FLOW_BIN_ARGS` also accepts whitespace-separated arguments. When using
|
|
104
|
+
the standalone executable, point `KSQL_FLOW_BIN` at it and leave
|
|
105
|
+
`KSQL_FLOW_BIN_ARGS` unset.
|
|
106
|
+
|
|
107
|
+
## Real-device E2E
|
|
108
|
+
|
|
109
|
+
After configuring the real-device environment described in
|
|
110
|
+
[`tests/e2e/README.md`](./tests/e2e/README.md), run the scenarios serially from
|
|
111
|
+
PowerShell. They access the configured kintone and kSQL-Flow environment and
|
|
112
|
+
must not run in CI. Each scenario writes a sanitized result JSON under
|
|
113
|
+
`tests/e2e/results/` and cleans up its own scoped state.
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
MIT. See [LICENSE](./LICENSE).
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { ApprovedInspectionException, InspectedNode } from "../executor/preflight.js";
|
|
2
|
+
export interface BundleJobInput {
|
|
3
|
+
readonly path: string;
|
|
4
|
+
readonly sqlBytes: Uint8Array;
|
|
5
|
+
/** Validated result returned by validateJobInspections. */
|
|
6
|
+
readonly inspectedNode: InspectedNode;
|
|
7
|
+
}
|
|
8
|
+
export interface BundleFileManifest {
|
|
9
|
+
readonly path: string;
|
|
10
|
+
readonly byteLength: number;
|
|
11
|
+
readonly sha256: string;
|
|
12
|
+
readonly nodeId?: string;
|
|
13
|
+
readonly jobId?: string;
|
|
14
|
+
readonly nondeterministicCodes?: readonly string[];
|
|
15
|
+
readonly approvedExceptions?: readonly ApprovedInspectionException[];
|
|
16
|
+
}
|
|
17
|
+
export interface BundleManifest {
|
|
18
|
+
readonly formatVersion: 1;
|
|
19
|
+
readonly kind: "EXECUTION_BUNDLE_MANIFEST";
|
|
20
|
+
readonly files: readonly BundleFileManifest[];
|
|
21
|
+
}
|
|
22
|
+
export interface BuildBundleInput {
|
|
23
|
+
readonly networkYamlBytes: Uint8Array;
|
|
24
|
+
readonly jobs: readonly BundleJobInput[];
|
|
25
|
+
}
|
|
26
|
+
export interface BuildBundleResult {
|
|
27
|
+
readonly zipBytes: Buffer;
|
|
28
|
+
readonly zipSha256: string;
|
|
29
|
+
readonly manifest: BundleManifest;
|
|
30
|
+
readonly manifestSha256: string;
|
|
31
|
+
}
|
|
32
|
+
export interface VerifyBundleExpectations {
|
|
33
|
+
readonly zipSha256?: string;
|
|
34
|
+
readonly manifestSha256?: string;
|
|
35
|
+
readonly manifest?: BundleManifest;
|
|
36
|
+
}
|
|
37
|
+
export interface VerifyBundleResult {
|
|
38
|
+
readonly zipSha256: string;
|
|
39
|
+
readonly manifestSha256: string;
|
|
40
|
+
readonly manifest: BundleManifest;
|
|
41
|
+
}
|
|
42
|
+
export type BundleErrorCode = "BUNDLE_INPUT_INVALID" | "BUNDLE_ZIP_INVALID" | "BUNDLE_HASH_MISMATCH" | "BUNDLE_MANIFEST_INVALID" | "BUNDLE_FILE_MISMATCH";
|
|
43
|
+
export declare class BundleError extends Error {
|
|
44
|
+
readonly code: BundleErrorCode;
|
|
45
|
+
constructor(code: BundleErrorCode, message: string, options?: ErrorOptions);
|
|
46
|
+
}
|
|
47
|
+
export declare function buildBundle(input: BuildBundleInput): BuildBundleResult;
|
|
48
|
+
export declare function verifyBundle(zipInput: Uint8Array, expected?: VerifyBundleExpectations): VerifyBundleResult;
|
|
49
|
+
//# sourceMappingURL=bundle-builder.d.ts.map
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { canonicalJson, sha256Hex } from "../executor/preflight.js";
|
|
2
|
+
import { createStoreZip, readStoreZip } from "./zip-store.js";
|
|
3
|
+
export class BundleError extends Error {
|
|
4
|
+
code;
|
|
5
|
+
constructor(code, message, options) {
|
|
6
|
+
super(message, options);
|
|
7
|
+
this.code = code;
|
|
8
|
+
this.name = "BundleError";
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
export function buildBundle(input) {
|
|
12
|
+
if (input.jobs.length === 0) {
|
|
13
|
+
throw new BundleError("BUNDLE_INPUT_INVALID", "execution bundle requires at least one SQL job");
|
|
14
|
+
}
|
|
15
|
+
const sortedJobs = [...input.jobs].sort((left, right) => compareText(left.path, right.path));
|
|
16
|
+
const seenPaths = new Set();
|
|
17
|
+
const files = [
|
|
18
|
+
fileManifest("network.yaml", input.networkYamlBytes),
|
|
19
|
+
];
|
|
20
|
+
const zipEntries = [
|
|
21
|
+
{ name: "network.yaml", data: input.networkYamlBytes },
|
|
22
|
+
];
|
|
23
|
+
for (const job of sortedJobs) {
|
|
24
|
+
validateJobInput(job, seenPaths);
|
|
25
|
+
const approvedExceptions = [...job.inspectedNode.approvedExceptions].sort((left, right) => compareText(exceptionSortKey(left), exceptionSortKey(right)));
|
|
26
|
+
files.push({
|
|
27
|
+
...fileManifest(job.path, job.sqlBytes),
|
|
28
|
+
nodeId: job.inspectedNode.nodeId,
|
|
29
|
+
jobId: job.inspectedNode.jobId,
|
|
30
|
+
nondeterministicCodes: [
|
|
31
|
+
...new Set(job.inspectedNode.nondeterministicCodes),
|
|
32
|
+
].sort(),
|
|
33
|
+
approvedExceptions,
|
|
34
|
+
});
|
|
35
|
+
zipEntries.push({ name: job.path, data: job.sqlBytes });
|
|
36
|
+
}
|
|
37
|
+
const manifest = {
|
|
38
|
+
formatVersion: 1,
|
|
39
|
+
kind: "EXECUTION_BUNDLE_MANIFEST",
|
|
40
|
+
files,
|
|
41
|
+
};
|
|
42
|
+
const manifestBytes = Buffer.from(canonicalJson(manifest), "utf8");
|
|
43
|
+
const zipBytes = createStoreZip([
|
|
44
|
+
...zipEntries,
|
|
45
|
+
{ name: "manifest.json", data: manifestBytes },
|
|
46
|
+
]);
|
|
47
|
+
return {
|
|
48
|
+
zipBytes,
|
|
49
|
+
zipSha256: sha256Hex(zipBytes),
|
|
50
|
+
manifest,
|
|
51
|
+
manifestSha256: sha256Hex(manifestBytes),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export function verifyBundle(zipInput, expected = {}) {
|
|
55
|
+
const zipBytes = Buffer.from(zipInput);
|
|
56
|
+
const zipSha256 = sha256Hex(zipBytes);
|
|
57
|
+
if (expected.zipSha256 !== undefined && zipSha256 !== expected.zipSha256) {
|
|
58
|
+
throw new BundleError("BUNDLE_HASH_MISMATCH", "execution bundle ZIP SHA-256 does not match");
|
|
59
|
+
}
|
|
60
|
+
let entries;
|
|
61
|
+
try {
|
|
62
|
+
entries = readStoreZip(zipBytes);
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
throw new BundleError("BUNDLE_ZIP_INVALID", "execution bundle ZIP is invalid", {
|
|
66
|
+
cause: error,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
const entryMap = new Map(entries.map((entry) => [entry.name, entry.data]));
|
|
70
|
+
const manifestBytes = entryMap.get("manifest.json");
|
|
71
|
+
if (manifestBytes === undefined) {
|
|
72
|
+
throw new BundleError("BUNDLE_MANIFEST_INVALID", "execution bundle has no manifest.json");
|
|
73
|
+
}
|
|
74
|
+
const manifestSha256 = sha256Hex(manifestBytes);
|
|
75
|
+
if (expected.manifestSha256 !== undefined &&
|
|
76
|
+
manifestSha256 !== expected.manifestSha256) {
|
|
77
|
+
throw new BundleError("BUNDLE_HASH_MISMATCH", "execution bundle manifest SHA-256 does not match");
|
|
78
|
+
}
|
|
79
|
+
let parsed;
|
|
80
|
+
try {
|
|
81
|
+
parsed = JSON.parse(manifestBytes.toString("utf8"));
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
throw new BundleError("BUNDLE_MANIFEST_INVALID", "execution bundle manifest is not valid JSON", { cause: error });
|
|
85
|
+
}
|
|
86
|
+
if (!isBundleManifest(parsed)) {
|
|
87
|
+
throw new BundleError("BUNDLE_MANIFEST_INVALID", "execution bundle manifest shape is invalid");
|
|
88
|
+
}
|
|
89
|
+
if (canonicalJson(parsed) !== manifestBytes.toString("utf8")) {
|
|
90
|
+
throw new BundleError("BUNDLE_MANIFEST_INVALID", "execution bundle manifest is not canonical JSON");
|
|
91
|
+
}
|
|
92
|
+
if (expected.manifest !== undefined &&
|
|
93
|
+
canonicalJson(parsed) !== canonicalJson(expected.manifest)) {
|
|
94
|
+
throw new BundleError("BUNDLE_MANIFEST_INVALID", "execution bundle manifest content does not match expected manifest");
|
|
95
|
+
}
|
|
96
|
+
const manifestPaths = new Set();
|
|
97
|
+
for (const file of parsed.files) {
|
|
98
|
+
if (manifestPaths.has(file.path)) {
|
|
99
|
+
throw new BundleError("BUNDLE_MANIFEST_INVALID", `manifest contains duplicate path '${file.path}'`);
|
|
100
|
+
}
|
|
101
|
+
manifestPaths.add(file.path);
|
|
102
|
+
const bytes = entryMap.get(file.path);
|
|
103
|
+
if (bytes === undefined ||
|
|
104
|
+
bytes.length !== file.byteLength ||
|
|
105
|
+
sha256Hex(bytes) !== file.sha256) {
|
|
106
|
+
throw new BundleError("BUNDLE_FILE_MISMATCH", `bundle file '${file.path}' does not match its manifest`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
if (!manifestPaths.has("network.yaml")) {
|
|
110
|
+
throw new BundleError("BUNDLE_MANIFEST_INVALID", "manifest does not contain network.yaml");
|
|
111
|
+
}
|
|
112
|
+
const unexpectedEntries = entries
|
|
113
|
+
.map((entry) => entry.name)
|
|
114
|
+
.filter((name) => name !== "manifest.json" && !manifestPaths.has(name));
|
|
115
|
+
if (unexpectedEntries.length > 0 ||
|
|
116
|
+
entries.length !== parsed.files.length + 1) {
|
|
117
|
+
throw new BundleError("BUNDLE_FILE_MISMATCH", `bundle has entries not represented by the manifest: ${unexpectedEntries.join(", ")}`);
|
|
118
|
+
}
|
|
119
|
+
return { zipSha256, manifestSha256, manifest: parsed };
|
|
120
|
+
}
|
|
121
|
+
function fileManifest(path, bytes) {
|
|
122
|
+
return { path, byteLength: bytes.byteLength, sha256: sha256Hex(bytes) };
|
|
123
|
+
}
|
|
124
|
+
function validateJobInput(job, seenPaths) {
|
|
125
|
+
if (!/^jobs\/[A-Za-z0-9._-]+\.sql$/u.test(job.path) ||
|
|
126
|
+
job.path.includes("..") ||
|
|
127
|
+
job.inspectedNode.nodeId.trim() === "" ||
|
|
128
|
+
job.inspectedNode.jobId.trim() === "") {
|
|
129
|
+
throw new BundleError("BUNDLE_INPUT_INVALID", `bundle job input is invalid for '${job.path}'`);
|
|
130
|
+
}
|
|
131
|
+
if (seenPaths.has(job.path)) {
|
|
132
|
+
throw new BundleError("BUNDLE_INPUT_INVALID", `bundle job path is duplicated: '${job.path}'`);
|
|
133
|
+
}
|
|
134
|
+
seenPaths.add(job.path);
|
|
135
|
+
if (job.inspectedNode.inspection.jobId !== job.inspectedNode.jobId) {
|
|
136
|
+
throw new BundleError("BUNDLE_INPUT_INVALID", `inspected job ID does not match preflight result for '${job.inspectedNode.nodeId}'`);
|
|
137
|
+
}
|
|
138
|
+
for (const exception of job.inspectedNode.approvedExceptions) {
|
|
139
|
+
if (exception.node_id !== job.inspectedNode.nodeId ||
|
|
140
|
+
!job.inspectedNode.nondeterministicCodes.includes(exception.code)) {
|
|
141
|
+
throw new BundleError("BUNDLE_INPUT_INVALID", `approved exception does not match inspected job '${job.inspectedNode.nodeId}'`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
function exceptionSortKey(exception) {
|
|
146
|
+
return [
|
|
147
|
+
exception.node_id,
|
|
148
|
+
exception.code,
|
|
149
|
+
exception.approved_by,
|
|
150
|
+
exception.approved_at,
|
|
151
|
+
exception.reason,
|
|
152
|
+
].join("\0");
|
|
153
|
+
}
|
|
154
|
+
function isRecord(value) {
|
|
155
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
156
|
+
}
|
|
157
|
+
function isBundleManifest(value) {
|
|
158
|
+
return (isRecord(value) &&
|
|
159
|
+
value.formatVersion === 1 &&
|
|
160
|
+
value.kind === "EXECUTION_BUNDLE_MANIFEST" &&
|
|
161
|
+
Array.isArray(value.files) &&
|
|
162
|
+
value.files.length > 0 &&
|
|
163
|
+
value.files.every(isBundleFileManifest));
|
|
164
|
+
}
|
|
165
|
+
function isBundleFileManifest(value) {
|
|
166
|
+
if (!isRecord(value) ||
|
|
167
|
+
typeof value.path !== "string" ||
|
|
168
|
+
!Number.isSafeInteger(value.byteLength) ||
|
|
169
|
+
value.byteLength < 0 ||
|
|
170
|
+
typeof value.sha256 !== "string" ||
|
|
171
|
+
!/^[a-f0-9]{64}$/u.test(value.sha256)) {
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
if (value.path === "network.yaml") {
|
|
175
|
+
return (value.nodeId === undefined &&
|
|
176
|
+
value.jobId === undefined &&
|
|
177
|
+
value.nondeterministicCodes === undefined &&
|
|
178
|
+
value.approvedExceptions === undefined);
|
|
179
|
+
}
|
|
180
|
+
if (!/^jobs\/[A-Za-z0-9._-]+\.sql$/u.test(value.path) ||
|
|
181
|
+
typeof value.nodeId !== "string" ||
|
|
182
|
+
typeof value.jobId !== "string" ||
|
|
183
|
+
!Array.isArray(value.nondeterministicCodes) ||
|
|
184
|
+
!value.nondeterministicCodes.every((code) => typeof code === "string") ||
|
|
185
|
+
!Array.isArray(value.approvedExceptions) ||
|
|
186
|
+
!value.approvedExceptions.every(isApprovedException)) {
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
const codes = value.nondeterministicCodes;
|
|
190
|
+
const exceptions = value.approvedExceptions;
|
|
191
|
+
return exceptions.every((exception) => exception.node_id === value.nodeId && codes.includes(exception.code));
|
|
192
|
+
}
|
|
193
|
+
function isApprovedException(value) {
|
|
194
|
+
return (isRecord(value) &&
|
|
195
|
+
typeof value.node_id === "string" &&
|
|
196
|
+
value.code === "KSQL1306" &&
|
|
197
|
+
typeof value.approved_by === "string" &&
|
|
198
|
+
typeof value.reason === "string" &&
|
|
199
|
+
typeof value.approved_at === "string");
|
|
200
|
+
}
|
|
201
|
+
function compareText(left, right) {
|
|
202
|
+
if (left < right)
|
|
203
|
+
return -1;
|
|
204
|
+
if (left > right)
|
|
205
|
+
return 1;
|
|
206
|
+
return 0;
|
|
207
|
+
}
|
|
208
|
+
//# sourceMappingURL=bundle-builder.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export type BundleFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
2
|
+
export interface UploadBundleOptions {
|
|
3
|
+
readonly endpoint: string;
|
|
4
|
+
readonly zipBytes: Uint8Array;
|
|
5
|
+
readonly fetch: BundleFetch;
|
|
6
|
+
readonly filename?: string;
|
|
7
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
8
|
+
}
|
|
9
|
+
export interface DownloadBundleOptions {
|
|
10
|
+
readonly endpoint: string;
|
|
11
|
+
readonly fileKey: string;
|
|
12
|
+
readonly fetch: BundleFetch;
|
|
13
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
14
|
+
}
|
|
15
|
+
export declare class BundleUploadError extends Error {
|
|
16
|
+
readonly code = "BUNDLE_UPLOAD_FAILED";
|
|
17
|
+
constructor(message: string, options?: ErrorOptions);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Uploads one bundle and returns the newly issued attachment fileKey.
|
|
21
|
+
*
|
|
22
|
+
* kintone upload fileKey values are single-use for attachment. Every attach or
|
|
23
|
+
* re-attach must call this helper again; callers must never cache or reuse a
|
|
24
|
+
* previously consumed fileKey (D-12).
|
|
25
|
+
*/
|
|
26
|
+
export declare function uploadBundle(options: UploadBundleOptions): Promise<string>;
|
|
27
|
+
export declare function downloadBundle(options: DownloadBundleOptions): Promise<Buffer>;
|
|
28
|
+
//# sourceMappingURL=upload.d.ts.map
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export class BundleUploadError extends Error {
|
|
2
|
+
code = "BUNDLE_UPLOAD_FAILED";
|
|
3
|
+
constructor(message, options) {
|
|
4
|
+
super(message, options);
|
|
5
|
+
this.name = "BundleUploadError";
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Uploads one bundle and returns the newly issued attachment fileKey.
|
|
10
|
+
*
|
|
11
|
+
* kintone upload fileKey values are single-use for attachment. Every attach or
|
|
12
|
+
* re-attach must call this helper again; callers must never cache or reuse a
|
|
13
|
+
* previously consumed fileKey (D-12).
|
|
14
|
+
*/
|
|
15
|
+
export async function uploadBundle(options) {
|
|
16
|
+
const body = new FormData();
|
|
17
|
+
body.append("file", new Blob([Buffer.from(options.zipBytes)], { type: "application/zip" }), options.filename ?? "execution-bundle.zip");
|
|
18
|
+
let response;
|
|
19
|
+
try {
|
|
20
|
+
response = await options.fetch(options.endpoint, {
|
|
21
|
+
method: "POST",
|
|
22
|
+
...(options.headers === undefined ? {} : { headers: options.headers }),
|
|
23
|
+
body,
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
throw new BundleUploadError("bundle upload request failed", {
|
|
28
|
+
cause: error,
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
if (!response.ok) {
|
|
32
|
+
throw new BundleUploadError(`bundle upload returned HTTP ${response.status}`);
|
|
33
|
+
}
|
|
34
|
+
let value;
|
|
35
|
+
try {
|
|
36
|
+
value = await response.json();
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
throw new BundleUploadError("bundle upload returned invalid JSON", {
|
|
40
|
+
cause: error,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
if (value === null ||
|
|
44
|
+
typeof value !== "object" ||
|
|
45
|
+
!("fileKey" in value) ||
|
|
46
|
+
typeof value.fileKey !== "string" ||
|
|
47
|
+
value.fileKey === "") {
|
|
48
|
+
throw new BundleUploadError("bundle upload response has no fileKey");
|
|
49
|
+
}
|
|
50
|
+
return value.fileKey;
|
|
51
|
+
}
|
|
52
|
+
export async function downloadBundle(options) {
|
|
53
|
+
const url = new URL(options.endpoint);
|
|
54
|
+
url.searchParams.set("fileKey", options.fileKey);
|
|
55
|
+
let response;
|
|
56
|
+
try {
|
|
57
|
+
response = await options.fetch(url, {
|
|
58
|
+
method: "GET",
|
|
59
|
+
...(options.headers === undefined ? {} : { headers: options.headers }),
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
throw new BundleUploadError("bundle download request failed", {
|
|
64
|
+
cause: error,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
if (!response.ok) {
|
|
68
|
+
throw new BundleUploadError(`bundle download returned HTTP ${response.status}`);
|
|
69
|
+
}
|
|
70
|
+
return Buffer.from(await response.arrayBuffer());
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=upload.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface StoreZipEntry {
|
|
2
|
+
readonly name: string;
|
|
3
|
+
readonly data: Uint8Array;
|
|
4
|
+
}
|
|
5
|
+
export interface ReadStoreZipEntry {
|
|
6
|
+
readonly name: string;
|
|
7
|
+
readonly data: Buffer;
|
|
8
|
+
readonly crc32: number;
|
|
9
|
+
}
|
|
10
|
+
export declare class StoreZipError extends Error {
|
|
11
|
+
constructor(message: string);
|
|
12
|
+
}
|
|
13
|
+
/** Store-only ZIP with fixed zero DOS timestamps and caller-defined entry order. */
|
|
14
|
+
export declare function createStoreZip(entries: readonly StoreZipEntry[]): Buffer;
|
|
15
|
+
export declare function readStoreZip(input: Uint8Array): readonly ReadStoreZipEntry[];
|
|
16
|
+
export declare function crc32(input: Uint8Array): number;
|
|
17
|
+
//# sourceMappingURL=zip-store.d.ts.map
|