@citeark/agent 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +128 -0
- package/data/dataset-source-registry.v1.json +300 -0
- package/dist/arkgraph/boot.js +6 -0
- package/dist/arkgraph/index.html +1 -0
- package/dist/arkgraph/viewer.css +1 -0
- package/dist/arkgraph/viewer.en.css +1 -0
- package/dist/arkgraph/viewer.en.js +49 -0
- package/dist/arkgraph/viewer.en.js.LEGAL.txt +56 -0
- package/dist/arkgraph/viewer.js +49 -0
- package/dist/arkgraph/viewer.js.LEGAL.txt +56 -0
- package/docker/claude-code/Dockerfile +97 -0
- package/docker/claude-code/codex-pro-relay.mjs +466 -0
- package/docker/claude-code/runtime-contract-check.mjs +79 -0
- package/docs/arkgraph-reading.md +79 -0
- package/docs/configuration.md +100 -0
- package/docs/integration.md +92 -0
- package/docs/maturity-plan.md +27 -0
- package/docs/npm-release.md +44 -0
- package/docs/paper-reading.md +40 -0
- package/docs/research-plan-granularity.md +27 -0
- package/docs/terminal.md +49 -0
- package/examples/toy-evaluation/compile-task.json +27 -0
- package/examples/toy-evaluation/paper.md +5 -0
- package/examples/toy-evaluation/repository/README.md +9 -0
- package/examples/toy-evaluation/repository/checkpoint.json +4 -0
- package/examples/toy-evaluation/repository/evaluate.py +17 -0
- package/examples/toy-evaluation/task.json +81 -0
- package/package.json +59 -0
- package/prompts/compile-research.md +58 -0
- package/prompts/execute-contract.md +72 -0
- package/prompts/execute-workspace-simple.md +51 -0
- package/prompts/execute-workspace.md +34 -0
- package/prompts/prepare-reproduction.md +82 -0
- package/prompts/repair-research.md +45 -0
- package/protocol/CAP.md +129 -0
- package/protocol/LICENSE +12 -0
- package/protocol/MAPPINGS.md +72 -0
- package/protocol/README.md +38 -0
- package/protocol/conformance-v2.0-alpha.1.json +36 -0
- package/protocol/examples/arkgraph/checkpoint-evaluation.json +309 -0
- package/protocol/examples/arkgraph/fixtures.mjs +49 -0
- package/protocol/examples/arkgraph/paper-free.json +291 -0
- package/protocol/examples/arkgraph/partial-failure.json +344 -0
- package/protocol/examples/arkgraph/training-evaluation.json +443 -0
- package/protocol/profiles/agent-trace.md +16 -0
- package/protocol/profiles/computational-run.md +16 -0
- package/protocol/profiles/core.md +15 -0
- package/protocol/profiles/public-bundle.md +18 -0
- package/protocol/profiles/reproduction.md +29 -0
- package/protocol/profiles/research-compilation.md +44 -0
- package/protocol/profiles/research-plan.md +39 -0
- package/protocol/profiles/restricted-evidence.md +15 -0
- package/runtime/bootstrap-autodl-runtime.sh +314 -0
- package/runtime/create-runtime-venv.sh +41 -0
- package/runtime/install-local-cpu-runtime.sh +23 -0
- package/runtime/install-scientific-runtime.sh +153 -0
- package/runtime/mineru/parse.py +62 -0
- package/runtime/mineru/requirements.txt +4 -0
- package/runtime/requirements-baseline.txt +38 -0
- package/schemas/cap/v2/activity.schema.json +47 -0
- package/schemas/cap/v2/agent.schema.json +32 -0
- package/schemas/cap/v2/assertion.schema.json +110 -0
- package/schemas/cap/v2/descriptor.schema.json +243 -0
- package/schemas/cap/v2/entity.schema.json +64 -0
- package/schemas/cap/v2/manifest.schema.json +67 -0
- package/schemas/cap/v2/relation.schema.json +82 -0
- package/schemas/compute-catalog.schema.json +63 -0
- package/schemas/compute-decision.schema.json +27 -0
- package/schemas/execution-contract.schema.json +1024 -0
- package/schemas/research-card.schema.json +30 -0
- package/schemas/research-inventory-draft.schema.json +366 -0
- package/schemas/research.schema.json +1044 -0
- package/schemas/result.schema.json +173 -0
- package/schemas/verification-policy.schema.json +47 -0
- package/schemas/verified-conclusion.schema.json +58 -0
- package/schemas/workspace-summary.schema.json +24 -0
- package/scripts/build-arkgraph-view.mjs +12 -0
- package/scripts/check-execution-feasibility.mjs +24 -0
- package/scripts/check-syntax.mjs +15 -0
- package/scripts/deterministic-asset-preparation.py +438 -0
- package/scripts/package-cap.mjs +23 -0
- package/scripts/package-local-agent.mjs +23 -0
- package/scripts/preview-arkgraph.mjs +25 -0
- package/scripts/replay-research-compiler-candidate.mjs +134 -0
- package/scripts/review-compiler-sources.mjs +44 -0
- package/scripts/run-asset-preparation.sh +17 -0
- package/scripts/run-research-plan.mjs +98 -0
- package/scripts/validate-asset-preparation.py +290 -0
- package/scripts/verify-local-runtime.mjs +57 -0
- package/scripts/verify-npm-package.mjs +57 -0
- package/src/adapters/paper2agent.mjs +107 -0
- package/src/assets/cache.mjs +159 -0
- package/src/assets/compute.mjs +98 -0
- package/src/assets/executor.mjs +145 -0
- package/src/assets/lifecycle.mjs +213 -0
- package/src/assets/manifest.mjs +242 -0
- package/src/assets/opportunistic-preparation.mjs +81 -0
- package/src/assets/plan.mjs +411 -0
- package/src/assets/prompts.mjs +29 -0
- package/src/assets/public-asset-probe.mjs +525 -0
- package/src/assets/qualification.mjs +119 -0
- package/src/assets/readiness.mjs +130 -0
- package/src/assets/reproduction-admission.mjs +355 -0
- package/src/assets/requirements.mjs +152 -0
- package/src/assets/source-grounding.mjs +341 -0
- package/src/assets/source-policy.mjs +118 -0
- package/src/autodl/client.mjs +260 -0
- package/src/autodl/ssh.mjs +380 -0
- package/src/autodl/tools.mjs +129 -0
- package/src/cap/redaction.mjs +38 -0
- package/src/cap/v2/archive.mjs +152 -0
- package/src/cap/v2/attestation.mjs +204 -0
- package/src/cap/v2/canonical-json.mjs +114 -0
- package/src/cap/v2/compilation-artifact.mjs +240 -0
- package/src/cap/v2/core.mjs +282 -0
- package/src/cap/v2/measurement-assessment-records.mjs +23 -0
- package/src/cap/v2/pipeline-artifact.mjs +922 -0
- package/src/cap/v2/read.mjs +41 -0
- package/src/cap/v2/reassessment-artifact.mjs +383 -0
- package/src/cap/v2/research-artifact.mjs +231 -0
- package/src/cap/v2/research-map-records.mjs +46 -0
- package/src/cap/v2/research-object-records.mjs +163 -0
- package/src/cap/v2/research-records.mjs +187 -0
- package/src/cap/v2/verify.mjs +642 -0
- package/src/cli.mjs +1146 -0
- package/src/compute/autodl-pro-compiler.mjs +347 -0
- package/src/compute/autodl-pro-executor.mjs +459 -0
- package/src/compute/autodl-pro-job.mjs +843 -0
- package/src/compute/autodl-pro-network.mjs +295 -0
- package/src/compute/autodl-pro-remote.mjs +810 -0
- package/src/compute/autodl-pro-staging.mjs +117 -0
- package/src/compute/campaign.mjs +110 -0
- package/src/compute/catalog.mjs +123 -0
- package/src/compute/checkpoint-protocol.mjs +154 -0
- package/src/compute/codex-account-lock.mjs +111 -0
- package/src/compute/codex-account-session.mjs +107 -0
- package/src/compute/compiler-profile.mjs +38 -0
- package/src/compute/compiler-router.mjs +23 -0
- package/src/compute/coordinator-recovery.mjs +210 -0
- package/src/compute/executor-router.mjs +29 -0
- package/src/compute/gcp-batch-compiler.mjs +685 -0
- package/src/compute/gcp-batch-executor.mjs +1215 -0
- package/src/compute/gcp-batch-failure.mjs +92 -0
- package/src/compute/gcp-batch-job.mjs +527 -0
- package/src/compute/gcp-batch-lifecycle.mjs +81 -0
- package/src/compute/gcp-checkpoint-worker.mjs +1633 -0
- package/src/compute/local-codex-compiler.mjs +52 -0
- package/src/compute/measurement-hardware.mjs +128 -0
- package/src/compute/remote-attempt.mjs +226 -0
- package/src/compute/requirements.mjs +124 -0
- package/src/compute/research-phases.mjs +48 -0
- package/src/compute/scheduler.mjs +452 -0
- package/src/compute/shared-workloads.mjs +26 -0
- package/src/compute/stage-archive.mjs +79 -0
- package/src/contracts/campaign-contract.mjs +52 -0
- package/src/contracts/execution-contract.mjs +819 -0
- package/src/contracts/execution-mode.mjs +19 -0
- package/src/contracts/execution-timeouts.mjs +45 -0
- package/src/contracts/execution-workload.mjs +68 -0
- package/src/contracts/preflight-schema.mjs +25 -0
- package/src/contracts/public-contract.mjs +63 -0
- package/src/contracts/subject-tags.mjs +31 -0
- package/src/dashboard/data.mjs +898 -0
- package/src/dashboard/server.mjs +79 -0
- package/src/dashboard/static/dashboard.css +366 -0
- package/src/dashboard/static/dashboard.js +560 -0
- package/src/dashboard/static/index.html +85 -0
- package/src/deployment/community-policy.mjs +9 -0
- package/src/deployment/environment.mjs +112 -0
- package/src/deployment/guided.mjs +98 -0
- package/src/deployment/handoff.mjs +102 -0
- package/src/deployment/local-contract.mjs +31 -0
- package/src/deployment/local.mjs +100 -0
- package/src/deployment/prepare.mjs +46 -0
- package/src/deployment/recipe.mjs +108 -0
- package/src/deployment/supplement.mjs +51 -0
- package/src/deployment/terminal.mjs +43 -0
- package/src/diagnosis/renderer.mjs +75 -0
- package/src/diagnosis/target-failure.mjs +46 -0
- package/src/evidence/parser-registry.mjs +54 -0
- package/src/evidence/parsers/fasttext-classification.mjs +82 -0
- package/src/evidence/parsers/json-scalar.mjs +96 -0
- package/src/evidence/parsers/simcse-senteval.mjs +104 -0
- package/src/evidence/parsers/starspace-classification.mjs +78 -0
- package/src/evidence/registry.mjs +147 -0
- package/src/execution/runner-audit.mjs +473 -0
- package/src/gcp/auth.mjs +106 -0
- package/src/gcp/batch-client.mjs +120 -0
- package/src/gcp/resource-discovery.mjs +177 -0
- package/src/gcp/rest.mjs +82 -0
- package/src/gcp/secret-manager.mjs +34 -0
- package/src/gcp/signed-url.mjs +133 -0
- package/src/gcp/storage.mjs +220 -0
- package/src/graph/command.mjs +41 -0
- package/src/graph/execution.mjs +97 -0
- package/src/graph/model.mjs +37 -0
- package/src/graph/presentation.mjs +110 -0
- package/src/graph/query.mjs +159 -0
- package/src/graph/research-relations.mjs +69 -0
- package/src/graph/source-page.mjs +12 -0
- package/src/graph/source-preview.mjs +34 -0
- package/src/graph/validate.mjs +76 -0
- package/src/job.mjs +496 -0
- package/src/network/autodl-routing-proxy.mjs +462 -0
- package/src/network/egress-proxy.mjs +158 -0
- package/src/observability/event-contract.mjs +230 -0
- package/src/observability/pipeline-monitor.mjs +166 -0
- package/src/pipeline/orchestrator.mjs +1281 -0
- package/src/pipeline/recovery-error.mjs +11 -0
- package/src/pipeline/replay.mjs +304 -0
- package/src/pipeline/shared-execution.mjs +115 -0
- package/src/pipeline/stage-checkpoint.mjs +86 -0
- package/src/pipeline/stage-recovery.mjs +101 -0
- package/src/pipeline/targets.mjs +110 -0
- package/src/process.mjs +143 -0
- package/src/protocol.mjs +312 -0
- package/src/provider/codex-account.mjs +44 -0
- package/src/provider/codex-completion.mjs +49 -0
- package/src/provider/completion.mjs +292 -0
- package/src/provider/model-client.mjs +44 -0
- package/src/provider/model-route.mjs +29 -0
- package/src/provider/openrouter-readiness.mjs +189 -0
- package/src/provider/reader-bridge.mjs +35 -0
- package/src/provider/relay.mjs +263 -0
- package/src/provider/runtime-auth.mjs +40 -0
- package/src/public/cap.d.mts +90 -0
- package/src/public/cap.mjs +12 -0
- package/src/public/contracts.d.mts +2 -0
- package/src/public/host.mjs +171 -0
- package/src/public/operations.d.mts +11 -0
- package/src/public/presentation.d.mts +4 -0
- package/src/records/views.mjs +26 -0
- package/src/remote/command.mjs +178 -0
- package/src/remote/ssh.mjs +59 -0
- package/src/repository-origin.mjs +81 -0
- package/src/reproduction/evidence-feedback.mjs +96 -0
- package/src/reproduction/incomplete-initialization.mjs +25 -0
- package/src/reproduction/lifecycle.mjs +253 -0
- package/src/reproduction/plan.mjs +132 -0
- package/src/reproduction/prompts.mjs +70 -0
- package/src/reproduction/runner.mjs +188 -0
- package/src/reproduction/summary.mjs +130 -0
- package/src/reproduction/workspace-mode.mjs +7 -0
- package/src/research/automatic-admission.mjs +156 -0
- package/src/research/compiler-coverage.mjs +85 -0
- package/src/research/compiler-failure.mjs +24 -0
- package/src/research/compiler-normalization-guards.mjs +112 -0
- package/src/research/compiler-repair.mjs +3 -0
- package/src/research/compiler.mjs +853 -0
- package/src/research/continuation-selection.mjs +26 -0
- package/src/research/execution-graph-context.mjs +43 -0
- package/src/research/experiment-importance.mjs +15 -0
- package/src/research/inventory-handoff.mjs +104 -0
- package/src/research/inventory-revisions.mjs +32 -0
- package/src/research/mineru-local.mjs +73 -0
- package/src/research/paper-command.mjs +19 -0
- package/src/research/paper-markdown.mjs +180 -0
- package/src/research/paper-source-map.mjs +69 -0
- package/src/research/planning-policy.mjs +88 -0
- package/src/research/reference-materials.mjs +11 -0
- package/src/research/reproduction-scope.mjs +30 -0
- package/src/research/research-map.mjs +94 -0
- package/src/research/research-objects.mjs +88 -0
- package/src/research/source-discovery.mjs +646 -0
- package/src/research/source-observations.mjs +75 -0
- package/src/research/source-review-cli-mcp.mjs +26 -0
- package/src/research/source-review-input.mjs +209 -0
- package/src/research/source-review-local-codex.mjs +36 -0
- package/src/research/source-review-model.mjs +70 -0
- package/src/research/source-review.mjs +173 -0
- package/src/research/structure.mjs +3163 -0
- package/src/research-card/renderer.mjs +277 -0
- package/src/research-card/verified-conclusion.mjs +143 -0
- package/src/results/output-registry.mjs +183 -0
- package/src/runtime/claude-code.mjs +52 -0
- package/src/runtime/codex-capacity-retry.mjs +87 -0
- package/src/runtime/codex.mjs +64 -0
- package/src/runtime/config.mjs +157 -0
- package/src/runtime/final-output.mjs +40 -0
- package/src/runtime/index.mjs +21 -0
- package/src/runtime/local-codex.mjs +74 -0
- package/src/runtime/opencode.mjs +95 -0
- package/src/runtime/prompt.mjs +13 -0
- package/src/sandbox/docker.mjs +363 -0
- package/src/settings/command.mjs +297 -0
- package/src/settings/store.mjs +119 -0
- package/src/telemetry/pricing.mjs +68 -0
- package/src/telemetry/usage.mjs +265 -0
- package/src/terminal/events.mjs +97 -0
- package/src/terminal/input.mjs +40 -0
- package/src/terminal/plain.mjs +40 -0
- package/src/terminal/remote-stream.mjs +22 -0
- package/src/terminal/screen.mjs +214 -0
- package/src/terminal/transcript.mjs +69 -0
- package/src/util.mjs +107 -0
- package/src/verification/ai-assessor.mjs +534 -0
- package/src/verification/claim-evaluator.mjs +242 -0
- package/src/verification/evidence-context.mjs +165 -0
- package/src/verification/evidence-reader.mjs +95 -0
- package/src/verification/integrity.mjs +570 -0
- package/src/verification/tolerance.mjs +32 -0
- package/src/workloads/cpu-research-preparation.mjs +56 -0
- package/src/workloads/definition.mjs +74 -0
- package/src/workloads/phase-aware-reproduction.mjs +46 -0
- package/src/workloads/reproduction.mjs +85 -0
- package/src/workspace/command.mjs +242 -0
- package/src/workspace/control.mjs +49 -0
- package/src/workspace/entry.mjs +28 -0
- package/src/workspace/input.mjs +93 -0
- package/src/workspace/interactive.mjs +94 -0
- package/src/workspace/jobs.mjs +418 -0
- package/src/workspace/session.mjs +97 -0
- package/src/workspace/worker.mjs +137 -0
- package/ui/arkgraph/ambient-motion.mjs +10 -0
- package/ui/arkgraph/app.jsx +153 -0
- package/ui/arkgraph/boot.js +6 -0
- package/ui/arkgraph/camera-motion.mjs +20 -0
- package/ui/arkgraph/context-reveal.mjs +39 -0
- package/ui/arkgraph/details.css +3 -0
- package/ui/arkgraph/entry.jsx +28 -0
- package/ui/arkgraph/experiment-curves.mjs +17 -0
- package/ui/arkgraph/experiment-selection.mjs +15 -0
- package/ui/arkgraph/experiment-style.css +26 -0
- package/ui/arkgraph/experiment-ui.jsx +32 -0
- package/ui/arkgraph/frame.html +1 -0
- package/ui/arkgraph/graph-gestures.mjs +62 -0
- package/ui/arkgraph/label-layout.mjs +57 -0
- package/ui/arkgraph/locales/en.json +229 -0
- package/ui/arkgraph/locales/source-types.json +15 -0
- package/ui/arkgraph/localization-build.mjs +27 -0
- package/ui/arkgraph/material-build.mjs +23 -0
- package/ui/arkgraph/material-colors.mjs +39 -0
- package/ui/arkgraph/material-style.css +15 -0
- package/ui/arkgraph/open-graph.jsx +326 -0
- package/ui/arkgraph/outline.jsx +49 -0
- package/ui/arkgraph/package-lock.json +888 -0
- package/ui/arkgraph/package.json +17 -0
- package/ui/arkgraph/reading-layout.mjs +130 -0
- package/ui/arkgraph/reading-presentation.mjs +73 -0
- package/ui/arkgraph/record-detail.css +51 -0
- package/ui/arkgraph/record-details.jsx +29 -0
- package/ui/arkgraph/research-types.mjs +31 -0
- package/ui/arkgraph/selection-mark.jsx +6 -0
- package/ui/arkgraph/soft-spine.mjs +26 -0
- package/ui/arkgraph/steering-style.css +187 -0
- package/ui/arkgraph/style.css +272 -0
package/src/gcp/auth.mjs
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { captureProcess } from "../process.mjs";
|
|
2
|
+
import { CiteArkError } from "../util.mjs";
|
|
3
|
+
import { executeObservedExternalRequest } from "../observability/event-contract.mjs";
|
|
4
|
+
|
|
5
|
+
const METADATA_TOKEN_URL = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token";
|
|
6
|
+
// accessToken() refreshes cached credentials 60 seconds early. A one-minute
|
|
7
|
+
// gcloud cache therefore had an effective lifetime of zero and retried the
|
|
8
|
+
// unavailable GCE metadata endpoint before every local API request.
|
|
9
|
+
const GCLOUD_TOKEN_CACHE_MS = 5 * 60_000;
|
|
10
|
+
|
|
11
|
+
export class GoogleAccessTokenProvider {
|
|
12
|
+
constructor({
|
|
13
|
+
accessToken,
|
|
14
|
+
fetchImpl = fetch,
|
|
15
|
+
captureProcessImpl = captureProcess,
|
|
16
|
+
now = () => Date.now(),
|
|
17
|
+
observe,
|
|
18
|
+
correlation,
|
|
19
|
+
} = {}) {
|
|
20
|
+
this.explicitAccessToken = nonEmpty(accessToken) ?? nonEmpty(process.env.GOOGLE_OAUTH_ACCESS_TOKEN);
|
|
21
|
+
this.fetchImpl = fetchImpl;
|
|
22
|
+
this.captureProcessImpl = captureProcessImpl;
|
|
23
|
+
this.now = now;
|
|
24
|
+
this.observe = observe;
|
|
25
|
+
this.correlation = correlation;
|
|
26
|
+
this.cached = null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async headers() {
|
|
30
|
+
return { Authorization: `Bearer ${await this.accessToken()}` };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
invalidate() {
|
|
34
|
+
this.cached = null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async accessToken() {
|
|
38
|
+
if (this.explicitAccessToken) return this.explicitAccessToken;
|
|
39
|
+
if (this.cached && this.cached.expiresAt - 60_000 > this.now()) return this.cached.value;
|
|
40
|
+
|
|
41
|
+
const metadata = await this.#fromMetadata();
|
|
42
|
+
if (metadata) {
|
|
43
|
+
this.cached = metadata;
|
|
44
|
+
return metadata.value;
|
|
45
|
+
}
|
|
46
|
+
const gcloud = await this.#fromGcloud();
|
|
47
|
+
if (gcloud) {
|
|
48
|
+
// `gcloud auth print-access-token` does not report the token's actual
|
|
49
|
+
// remaining lifetime. Keep this cache deliberately short and let the
|
|
50
|
+
// REST layer invalidate it immediately when Google returns 401.
|
|
51
|
+
this.cached = { value: gcloud, expiresAt: this.now() + GCLOUD_TOKEN_CACHE_MS };
|
|
52
|
+
return gcloud;
|
|
53
|
+
}
|
|
54
|
+
throw new CiteArkError(
|
|
55
|
+
"无法取得 Google Cloud 访问令牌。请在 GCP 运行身份中执行,或先运行 gcloud auth application-default login。",
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async #fromMetadata() {
|
|
60
|
+
const controller = new AbortController();
|
|
61
|
+
const timeout = setTimeout(() => controller.abort(), 1_000);
|
|
62
|
+
timeout.unref?.();
|
|
63
|
+
try {
|
|
64
|
+
const response = await executeObservedExternalRequest({
|
|
65
|
+
request: async () => await this.fetchImpl(METADATA_TOKEN_URL, {
|
|
66
|
+
headers: { "Metadata-Flavor": "Google" },
|
|
67
|
+
signal: controller.signal,
|
|
68
|
+
}),
|
|
69
|
+
observe: this.observe,
|
|
70
|
+
component: "gcp_auth",
|
|
71
|
+
service: "metadata.google.internal",
|
|
72
|
+
operation: "service_account_token",
|
|
73
|
+
url: METADATA_TOKEN_URL,
|
|
74
|
+
correlation: this.correlation,
|
|
75
|
+
});
|
|
76
|
+
if (!response.ok) return null;
|
|
77
|
+
const payload = await response.json();
|
|
78
|
+
if (!nonEmpty(payload?.access_token)) return null;
|
|
79
|
+
return {
|
|
80
|
+
value: payload.access_token,
|
|
81
|
+
expiresAt: this.now() + Math.max(60, Number(payload.expires_in) || 3600) * 1000,
|
|
82
|
+
};
|
|
83
|
+
} catch {
|
|
84
|
+
return null;
|
|
85
|
+
} finally {
|
|
86
|
+
clearTimeout(timeout);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async #fromGcloud() {
|
|
91
|
+
const environment = { ...process.env, CLOUDSDK_CORE_DISABLE_PROMPTS: "1" };
|
|
92
|
+
for (const args of [
|
|
93
|
+
["auth", "application-default", "print-access-token"],
|
|
94
|
+
["auth", "print-access-token"],
|
|
95
|
+
]) {
|
|
96
|
+
const result = await this.captureProcessImpl("gcloud", args, { env: environment }).catch(() => null);
|
|
97
|
+
const token = nonEmpty(result?.code === 0 ? result.stdout : null);
|
|
98
|
+
if (token) return token;
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function nonEmpty(value) {
|
|
105
|
+
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
106
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { watchRemoteTranscript } from '../terminal/remote-stream.mjs';
|
|
2
|
+
import { googleJsonRequest } from "./rest.mjs";
|
|
3
|
+
import { CiteArkError } from "../util.mjs";
|
|
4
|
+
|
|
5
|
+
const TERMINAL_STATES = new Set(["SUCCEEDED", "FAILED", "CANCELLED"]);
|
|
6
|
+
|
|
7
|
+
export class GcpBatchClient {
|
|
8
|
+
constructor({
|
|
9
|
+
projectId,
|
|
10
|
+
region,
|
|
11
|
+
auth,
|
|
12
|
+
fetchImpl = fetch,
|
|
13
|
+
sleep = defaultSleep,
|
|
14
|
+
observe,
|
|
15
|
+
correlation,
|
|
16
|
+
budget,
|
|
17
|
+
} = {}) {
|
|
18
|
+
this.projectId = required(projectId, "projectId");
|
|
19
|
+
this.region = required(region, "region");
|
|
20
|
+
this.auth = auth;
|
|
21
|
+
this.fetchImpl = fetchImpl;
|
|
22
|
+
this.sleep = sleep;
|
|
23
|
+
this.observe = observe;
|
|
24
|
+
this.correlation = correlation;
|
|
25
|
+
this.budget = budget;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
jobName(jobId) {
|
|
29
|
+
return `projects/${this.projectId}/locations/${this.region}/jobs/${jobId}`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async createJob(jobId, job) {
|
|
33
|
+
await this.budget?.beforeCreate(jobId, job, { projectId: this.projectId, region: this.region });
|
|
34
|
+
const url = new URL(`https://batch.googleapis.com/v1/projects/${encodeURIComponent(this.projectId)}/locations/${encodeURIComponent(this.region)}/jobs`);
|
|
35
|
+
url.searchParams.set("job_id", jobId);
|
|
36
|
+
return await googleJsonRequest({
|
|
37
|
+
url,
|
|
38
|
+
auth: this.auth,
|
|
39
|
+
fetchImpl: this.fetchImpl,
|
|
40
|
+
method: "POST",
|
|
41
|
+
body: job,
|
|
42
|
+
observe: this.observe,
|
|
43
|
+
operation: "batch_create_job",
|
|
44
|
+
correlation: { ...this.correlation, gcpJobId: jobId },
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async findJob(name) {
|
|
49
|
+
const job = await googleJsonRequest({ url: `https://batch.googleapis.com/v1/${name}`, auth: this.auth,
|
|
50
|
+
fetchImpl: this.fetchImpl, acceptedStatuses: [404], observe: this.observe, operation: "batch_find_recovery_job",
|
|
51
|
+
correlation: { ...this.correlation, gcpJobName: name } });
|
|
52
|
+
if (job?.error?.code === 404) return null;
|
|
53
|
+
if (job?.name) await this.budget?.adopt?.(job);
|
|
54
|
+
return job;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async getJob(name) {
|
|
58
|
+
return await googleJsonRequest({
|
|
59
|
+
url: `https://batch.googleapis.com/v1/${name}`,
|
|
60
|
+
auth: this.auth,
|
|
61
|
+
fetchImpl: this.fetchImpl,
|
|
62
|
+
observe: this.observe,
|
|
63
|
+
operation: "batch_get_job",
|
|
64
|
+
correlation: { ...this.correlation, gcpJobName: name },
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async deleteJob(name) {
|
|
69
|
+
return await googleJsonRequest({
|
|
70
|
+
url: `https://batch.googleapis.com/v1/${name}`,
|
|
71
|
+
auth: this.auth,
|
|
72
|
+
fetchImpl: this.fetchImpl,
|
|
73
|
+
method: "DELETE",
|
|
74
|
+
acceptedStatuses: [404],
|
|
75
|
+
observe: this.observe,
|
|
76
|
+
operation: "batch_delete_job",
|
|
77
|
+
correlation: { ...this.correlation, gcpJobName: name },
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async waitForTerminal(name, {
|
|
82
|
+
timeoutMs,
|
|
83
|
+
pollIntervalMs = 10_000,
|
|
84
|
+
onState = () => {},
|
|
85
|
+
onPoll = () => {},
|
|
86
|
+
liveOutput,
|
|
87
|
+
}) {
|
|
88
|
+
const stopOutput = liveOutput ? watchRemoteTranscript(liveOutput) : async () => {};
|
|
89
|
+
try {
|
|
90
|
+
const deadline = Date.now() + timeoutMs;
|
|
91
|
+
let lastState = null;
|
|
92
|
+
while (Date.now() < deadline) {
|
|
93
|
+
const job = await this.getJob(name);
|
|
94
|
+
const state = job?.status?.state ?? "STATE_UNSPECIFIED";
|
|
95
|
+
await onPoll(job, state);
|
|
96
|
+
if (state !== lastState) {
|
|
97
|
+
lastState = state;
|
|
98
|
+
await onState(state, job);
|
|
99
|
+
}
|
|
100
|
+
if (TERMINAL_STATES.has(state)) {
|
|
101
|
+
await this.budget?.onTerminal(job);
|
|
102
|
+
return job;
|
|
103
|
+
}
|
|
104
|
+
await this.budget?.onPoll?.(job, () => this.deleteJob(name));
|
|
105
|
+
await this.sleep(Math.min(pollIntervalMs, Math.max(0, deadline - Date.now())));
|
|
106
|
+
}
|
|
107
|
+
if (this.budget) await this.deleteJob(name);
|
|
108
|
+
throw new CiteArkError(`Timed out waiting for GCP Batch job: ${name}`);
|
|
109
|
+
} finally { await stopOutput(); }
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function required(value, field) {
|
|
114
|
+
if (typeof value !== "string" || !value.trim()) throw new CiteArkError(`GCP Batch 缺少 ${field}`);
|
|
115
|
+
return value.trim();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function defaultSleep(milliseconds) {
|
|
119
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
120
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { GoogleAccessTokenProvider } from "./auth.mjs";
|
|
2
|
+
import { googleJsonRequest } from "./rest.mjs";
|
|
3
|
+
import { sha256Value } from "../util.mjs";
|
|
4
|
+
|
|
5
|
+
export const GCP_DISCOVERY_READ_PERMISSIONS = [
|
|
6
|
+
"compute.machineTypes.list", "compute.acceleratorTypes.list",
|
|
7
|
+
"compute.regions.list", "compute.projects.get",
|
|
8
|
+
];
|
|
9
|
+
const GPU_SPECS = {
|
|
10
|
+
"nvidia-tesla-k80": ["K80", 12], "nvidia-tesla-p4": ["P4", 8],
|
|
11
|
+
"nvidia-tesla-p100": ["P100", 16], "nvidia-tesla-v100": ["V100", 16],
|
|
12
|
+
"nvidia-tesla-t4": ["T4", 16], "nvidia-l4": ["L4", 24],
|
|
13
|
+
"nvidia-tesla-a100": ["A100", 40], "nvidia-a100-80gb": ["A100 80GB", 80],
|
|
14
|
+
"nvidia-h100-80gb": ["H100", 80], "nvidia-h100-mega-80gb": ["H100 Mega", 80],
|
|
15
|
+
"nvidia-h200-141gb": ["H200", 141], "nvidia-b200": ["B200", 180],
|
|
16
|
+
};
|
|
17
|
+
const GPU_SPEC_SOURCE = "https://cloud.google.com/compute/docs/gpus";
|
|
18
|
+
const tail = (value) => String(value ?? "").split("/").at(-1);
|
|
19
|
+
|
|
20
|
+
// Discovery only reads provider metadata. Neither quota nor a successful IAM
|
|
21
|
+
// probe is a reservation, a runtime validation, or proof of physical capacity.
|
|
22
|
+
export async function discoverGcpResources({ projectId, auth,
|
|
23
|
+
fetchImpl = fetch, observe, now = () => Date.now(), ttlMs = 30 * 60_000 } = {}) {
|
|
24
|
+
if (!projectId || !/^[a-z][a-z0-9-]{4,61}[a-z0-9]$/.test(projectId)) throw new TypeError("资源发现需要 GCP projectId");
|
|
25
|
+
auth ??= new GoogleAccessTokenProvider({ fetchImpl, observe });
|
|
26
|
+
await auth.headers();
|
|
27
|
+
const startedAt = now();
|
|
28
|
+
const deadline = AbortSignal.timeout(180_000);
|
|
29
|
+
const request = (url, options = {}) => googleJsonRequest({ url, auth, observe,
|
|
30
|
+
fetchImpl: (target, init) => fetchImpl(target, { ...init,
|
|
31
|
+
signal: AbortSignal.any([deadline, AbortSignal.timeout(30_000)]) }),
|
|
32
|
+
operation: "compute_resource_discovery", ...options });
|
|
33
|
+
async function pages(url, extract) {
|
|
34
|
+
const rows = []; const warnings = []; const seen = new Set();
|
|
35
|
+
for (;;) {
|
|
36
|
+
const page = await request(url.toString());
|
|
37
|
+
rows.push(...extract(page));
|
|
38
|
+
if (page.warning) warnings.push(page.warning);
|
|
39
|
+
for (const scope of Object.values(page.items ?? {})) {
|
|
40
|
+
if (scope?.warning && scope.warning.code !== "NO_RESULTS_ON_PAGE") warnings.push(scope.warning);
|
|
41
|
+
}
|
|
42
|
+
if (!page.nextPageToken) return { rows, warnings };
|
|
43
|
+
if (seen.has(page.nextPageToken)) throw Error("资源发现分页游标重复");
|
|
44
|
+
seen.add(page.nextPageToken);
|
|
45
|
+
url.searchParams.set("pageToken", page.nextPageToken);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const base = `https://compute.googleapis.com/compute/v1/projects/${projectId}`;
|
|
49
|
+
const aggregated = (kind, fields) => {
|
|
50
|
+
const url = new URL(`${base}/aggregated/${kind}`);
|
|
51
|
+
url.searchParams.set("maxResults", "500");
|
|
52
|
+
url.searchParams.set("returnPartialSuccess", "true");
|
|
53
|
+
url.searchParams.set("fields", `nextPageToken,warning,items/*/${kind}(${fields}),items/*/warning`);
|
|
54
|
+
return pages(url, (page) => Object.values(page.items ?? {}).flatMap((scope) => scope[kind] ?? []));
|
|
55
|
+
};
|
|
56
|
+
const sources = {
|
|
57
|
+
machineTypes: () => aggregated("machineTypes", "name,guestCpus,memoryMb,zone,accelerators,architecture,deprecated"),
|
|
58
|
+
acceleratorTypes: () => aggregated("acceleratorTypes", "name,description,zone,maximumCardsPerInstance,deprecated"),
|
|
59
|
+
regions: () => pages(new URL(`${base}/regions?maxResults=500`), (page) => page.items ?? []),
|
|
60
|
+
projectQuotas: () => request(base),
|
|
61
|
+
permissions: () => request(`https://cloudresourcemanager.googleapis.com/v1/projects/${projectId}:testIamPermissions`, {
|
|
62
|
+
method: "POST", body: { permissions: [...GCP_DISCOVERY_READ_PERMISSIONS, "batch.jobs.create"] },
|
|
63
|
+
}),
|
|
64
|
+
prices: () => pages(new URL("https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus?currencyCode=USD&pageSize=5000"), (page) => page.skus ?? []),
|
|
65
|
+
};
|
|
66
|
+
const keys = Object.keys(sources);
|
|
67
|
+
const results = await Promise.allSettled(keys.map((key) => sources[key]()));
|
|
68
|
+
const data = {}; const checks = {};
|
|
69
|
+
results.forEach((result, index) => {
|
|
70
|
+
const key = keys[index];
|
|
71
|
+
if (result.status === "fulfilled") {
|
|
72
|
+
data[key] = result.value;
|
|
73
|
+
checks[key] = { status: result.value.warnings?.length ? "partial" : "ok", warnings: result.value.warnings ?? [] };
|
|
74
|
+
} else {
|
|
75
|
+
checks[key] = { status: "unknown", error: String(result.reason?.message ?? result.reason).slice(0, 1200) };
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
const machines = groupByZones(data.machineTypes?.rows ?? [], (m) => ({
|
|
79
|
+
name: m.name, cpuCores: m.guestCpus, memoryGb: m.memoryMb / 1024,
|
|
80
|
+
architecture: m.architecture ?? "unknown", deprecated: m.deprecated?.state ?? null,
|
|
81
|
+
accelerators: (m.accelerators ?? []).map((g) => ({ type: tail(g.guestAcceleratorType), count: Number(g.guestAcceleratorCount) })),
|
|
82
|
+
}));
|
|
83
|
+
const accelerators = groupByZones(data.acceleratorTypes?.rows ?? [], (a) => ({
|
|
84
|
+
name: a.name, description: a.description, maximumCardsPerInstance: a.maximumCardsPerInstance,
|
|
85
|
+
type: GPU_SPECS[a.name]?.[0] ?? a.name, vramGb: GPU_SPECS[a.name]?.[1] ?? null,
|
|
86
|
+
specificationSource: GPU_SPECS[a.name] ? GPU_SPEC_SOURCE : null,
|
|
87
|
+
deprecated: a.deprecated?.state ?? null,
|
|
88
|
+
}));
|
|
89
|
+
const permissions = data.permissions?.permissions ?? [];
|
|
90
|
+
const snapshot = {
|
|
91
|
+
schemaVersion: "0.1", kind: "citeark.compute-resource-discovery", provider: "gcp", projectId,
|
|
92
|
+
observedAt: new Date(startedAt).toISOString(), expiresAt: new Date(startedAt + ttlMs).toISOString(),
|
|
93
|
+
status: Object.values(checks).every((c) => c.status === "ok") ? "complete" : "partial",
|
|
94
|
+
sources: checks, capacity: { status: "unknown", reason: "Listing and quotas do not establish live capacity; allocation is confirmed only by the provider." },
|
|
95
|
+
inventory: { machines, accelerators },
|
|
96
|
+
regions: (data.regions?.rows ?? []).map((r) => ({ name: r.name, status: r.status,
|
|
97
|
+
zones: (r.zones ?? []).map(tail), quotas: normalizeQuotas(r.quotas) })),
|
|
98
|
+
projectQuotas: normalizeQuotas(data.projectQuotas?.quotas),
|
|
99
|
+
permissions: { status: checks.permissions.status, granted: permissions,
|
|
100
|
+
missing: checks.permissions.status === "ok" ? [...GCP_DISCOVERY_READ_PERMISSIONS, "batch.jobs.create"].filter((p) => !permissions.includes(p)) : [],
|
|
101
|
+
scope: "Caller permissions only; excludes service-agent, service-account, organization-policy and runtime checks." },
|
|
102
|
+
prices: normalizePrices(data.prices?.rows ?? [], startedAt),
|
|
103
|
+
};
|
|
104
|
+
return { ...snapshot, digest: `sha256:${sha256Value(snapshot)}` };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function groupByZones(rows, project) {
|
|
108
|
+
const groups = new Map();
|
|
109
|
+
for (const row of rows) {
|
|
110
|
+
const value = project(row); const key = JSON.stringify(value);
|
|
111
|
+
const entry = groups.get(key) ?? { ...value, zones: [] };
|
|
112
|
+
const zone = tail(row.zone);
|
|
113
|
+
if (zone && !entry.zones.includes(zone)) entry.zones.push(zone);
|
|
114
|
+
groups.set(key, entry);
|
|
115
|
+
}
|
|
116
|
+
return [...groups.values()].map((g) => ({ ...g, zones: g.zones.sort() })).sort((a, b) => a.name.localeCompare(b.name));
|
|
117
|
+
}
|
|
118
|
+
function normalizeQuotas(quotas = []) {
|
|
119
|
+
return quotas.map((q) => ({ metric: q.metric, limit: q.limit, usage: q.usage,
|
|
120
|
+
remaining: Number.isFinite(q.limit) && Number.isFinite(q.usage) ? Math.max(0, q.limit - q.usage) : null }));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Exact ordinary on-demand SKU descriptions avoid accidental Spot, commitment,
|
|
124
|
+
// DWS or sole-tenancy prices. Unknown and ambiguous prices stay unknown.
|
|
125
|
+
export function normalizePrices(skus, at = Date.now()) {
|
|
126
|
+
return skus.flatMap((sku) => {
|
|
127
|
+
if (sku.category?.usageType !== "OnDemand") return [];
|
|
128
|
+
const match = /^(E2 Instance|N2 Instance|N1 Predefined Instance|G2 Instance) (Core|Ram) running in /i.exec(sku.description ?? "");
|
|
129
|
+
const gpu = /^Nvidia (Tesla T4|L4) GPU running in /i.exec(sku.description ?? "");
|
|
130
|
+
if (!match && !gpu) return [];
|
|
131
|
+
const info = (sku.pricingInfo ?? []).filter((p) => !p.effectiveTime || Date.parse(p.effectiveTime) <= at)
|
|
132
|
+
.sort((a, b) => Date.parse(b.effectiveTime ?? 0) - Date.parse(a.effectiveTime ?? 0))[0];
|
|
133
|
+
const expression = info?.pricingExpression;
|
|
134
|
+
const tier = expression?.tieredRates?.length === 1 ? expression.tieredRates[0] : null;
|
|
135
|
+
if (!tier || Number(tier.startUsageAmount) !== 0 || tier.unitPrice?.currencyCode !== "USD") return [];
|
|
136
|
+
const component = gpu ? "gpu" : match[2].toLowerCase() === "core" ? "cpu" : "memory";
|
|
137
|
+
if (expression.usageUnit !== (component === "memory" ? "GiBy.h" : "h")) return [];
|
|
138
|
+
const unitUsd = Number(tier.unitPrice.units ?? 0) + Number(tier.unitPrice.nanos ?? 0) / 1e9;
|
|
139
|
+
if (!Number.isFinite(unitUsd) || unitUsd <= 0) return [];
|
|
140
|
+
return [{ skuId: sku.skuId, description: sku.description, regions: sku.serviceRegions ?? [], component,
|
|
141
|
+
family: gpu ? (gpu[1].toUpperCase().includes("T4") ? "T4" : "L4") : match[1].split(" ")[0].toLowerCase(),
|
|
142
|
+
unitUsd, effectiveTime: info.effectiveTime ?? null, source: "Google Cloud Billing Catalog API" }];
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export function discoveredGcpPrice(snapshot, { family, region, cpuCores, memoryGb, gpu }) {
|
|
147
|
+
const parts = [[family, "cpu", cpuCores], [family, "memory", memoryGb]];
|
|
148
|
+
if (gpu) parts.push([gpu.type, "gpu", gpu.count]);
|
|
149
|
+
const lines = [];
|
|
150
|
+
for (const [key, component, units] of parts) {
|
|
151
|
+
const prices = snapshot.prices.filter((p) => p.family === key && p.component === component && p.regions.includes(region));
|
|
152
|
+
if (prices.length !== 1 || !Number.isFinite(units) || units <= 0) return null;
|
|
153
|
+
lines.push({ skuId: prices[0].skuId, units, unitUsd: prices[0].unitUsd, effectiveTime: prices[0].effectiveTime });
|
|
154
|
+
}
|
|
155
|
+
return { hourlyUsd: lines.reduce((total, p) => total + p.units * p.unitUsd, 0), lines,
|
|
156
|
+
source: "Google Cloud Billing Catalog API", region, provisioningModel: "STANDARD", observedAt: snapshot.observedAt };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function gcpDiscoverySummary(snapshot) {
|
|
160
|
+
const families = new Map();
|
|
161
|
+
for (const m of snapshot.inventory.machines) {
|
|
162
|
+
const family = m.name.split("-")[0];
|
|
163
|
+
const f = families.get(family) ?? { family, machineTypes: 0, cpuCores: [Infinity, 0], memoryGb: [Infinity, 0], regions: new Set() };
|
|
164
|
+
f.machineTypes++;
|
|
165
|
+
f.cpuCores = [Math.min(f.cpuCores[0], m.cpuCores), Math.max(f.cpuCores[1], m.cpuCores)];
|
|
166
|
+
f.memoryGb = [Math.min(f.memoryGb[0], m.memoryGb), Math.max(f.memoryGb[1], m.memoryGb)];
|
|
167
|
+
m.zones.forEach((z) => f.regions.add(z.replace(/-[a-z]$/, "")));
|
|
168
|
+
families.set(family, f);
|
|
169
|
+
}
|
|
170
|
+
return { digest: snapshot.digest, observedAt: snapshot.observedAt, expiresAt: snapshot.expiresAt,
|
|
171
|
+
status: snapshot.status, projectId: snapshot.projectId, sources: snapshot.sources, capacity: snapshot.capacity,
|
|
172
|
+
permissions: snapshot.permissions, projectQuotas: snapshot.projectQuotas.filter(q => /^(CPUS|GPUS)_ALL_REGIONS$/.test(q.metric)),
|
|
173
|
+
machineFamilies: [...families.values()].map((f) => ({ ...f, regions: [...f.regions].sort() })),
|
|
174
|
+
accelerators: snapshot.inventory.accelerators.map(({ zones, ...a }) => ({ ...a, regions: [...new Set(zones.map((z) => z.replace(/-[a-z]$/, "")))].sort() })),
|
|
175
|
+
scope: "Provider inventory is broader than executable profiles. Missing price, runtime support, policy or quota prevents automatic admission; unknown live capacity is not a scientific blocker.",
|
|
176
|
+
};
|
|
177
|
+
}
|
package/src/gcp/rest.mjs
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { CiteArkError } from "../util.mjs";
|
|
2
|
+
import {
|
|
3
|
+
emitStructuredObservation,
|
|
4
|
+
executeObservedExternalRequest,
|
|
5
|
+
} from "../observability/event-contract.mjs";
|
|
6
|
+
|
|
7
|
+
export async function googleJsonRequest({
|
|
8
|
+
url,
|
|
9
|
+
auth,
|
|
10
|
+
fetchImpl = fetch,
|
|
11
|
+
method = "GET",
|
|
12
|
+
body,
|
|
13
|
+
acceptedStatuses = [],
|
|
14
|
+
observe = emitStructuredObservation,
|
|
15
|
+
operation = "google_cloud_api",
|
|
16
|
+
correlation,
|
|
17
|
+
}) {
|
|
18
|
+
let attempt = 0;
|
|
19
|
+
const request = async () => {
|
|
20
|
+
attempt += 1;
|
|
21
|
+
return executeObservedExternalRequest({
|
|
22
|
+
request: async () => await fetchImpl(url, {
|
|
23
|
+
method,
|
|
24
|
+
headers: {
|
|
25
|
+
...(await auth.headers()),
|
|
26
|
+
Accept: "application/json",
|
|
27
|
+
...(body === undefined ? {} : { "Content-Type": "application/json" }),
|
|
28
|
+
},
|
|
29
|
+
body: body === undefined ? undefined : JSON.stringify(body),
|
|
30
|
+
}),
|
|
31
|
+
observe,
|
|
32
|
+
component: "gcp_client",
|
|
33
|
+
service: new URL(url).hostname,
|
|
34
|
+
operation,
|
|
35
|
+
method,
|
|
36
|
+
url,
|
|
37
|
+
attempt,
|
|
38
|
+
maxAttempts: 2,
|
|
39
|
+
retryableStatus: (status) => status === 401
|
|
40
|
+
|| new Set([408, 425, 429, 500, 502, 503, 504]).has(status),
|
|
41
|
+
correlation,
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
let response = await request();
|
|
45
|
+
if (response.status === 401 && typeof auth.invalidate === "function") {
|
|
46
|
+
auth.invalidate();
|
|
47
|
+
await response.body?.cancel?.().catch?.(() => {});
|
|
48
|
+
response = await request();
|
|
49
|
+
}
|
|
50
|
+
if (!response.ok && !acceptedStatuses.includes(response.status)) {
|
|
51
|
+
throw await googleApiError(response, method, url);
|
|
52
|
+
}
|
|
53
|
+
if (response.status === 204 || response.headers.get("content-length") === "0") return null;
|
|
54
|
+
const text = await response.text();
|
|
55
|
+
if (!text) return null;
|
|
56
|
+
try {
|
|
57
|
+
return JSON.parse(text);
|
|
58
|
+
} catch (error) {
|
|
59
|
+
throw new CiteArkError(`Google Cloud API 返回了无效 JSON:${method} ${redactUrl(url)}`, { cause: error });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export async function googleApiError(response, method, url) {
|
|
64
|
+
let detail = "";
|
|
65
|
+
try {
|
|
66
|
+
const payload = await response.json();
|
|
67
|
+
detail = payload?.error?.message ?? payload?.message ?? "";
|
|
68
|
+
} catch {
|
|
69
|
+
detail = await response.text().catch(() => "");
|
|
70
|
+
}
|
|
71
|
+
return new CiteArkError(
|
|
72
|
+
`Google Cloud API 请求失败(${response.status}):${method} ${redactUrl(url)}${detail ? `\n${detail}` : ""}`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function redactUrl(value) {
|
|
77
|
+
const url = new URL(value);
|
|
78
|
+
for (const key of [...url.searchParams.keys()]) {
|
|
79
|
+
if (/token|key|secret/i.test(key)) url.searchParams.set(key, "<redacted>");
|
|
80
|
+
}
|
|
81
|
+
return url.toString();
|
|
82
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { GoogleAccessTokenProvider } from "./auth.mjs";
|
|
2
|
+
import { googleJsonRequest } from "./rest.mjs";
|
|
3
|
+
import { CiteArkError } from "../util.mjs";
|
|
4
|
+
|
|
5
|
+
export class GcpSecretManagerClient {
|
|
6
|
+
constructor({ auth, fetchImpl = fetch } = {}) {
|
|
7
|
+
this.auth = auth ?? new GoogleAccessTokenProvider();
|
|
8
|
+
this.fetchImpl = fetchImpl;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
async accessVersion(resource) {
|
|
12
|
+
const name = requiredSecretVersion(resource);
|
|
13
|
+
const payload = await googleJsonRequest({
|
|
14
|
+
url: `https://secretmanager.googleapis.com/v1/${name}:access`,
|
|
15
|
+
auth: this.auth,
|
|
16
|
+
fetchImpl: this.fetchImpl,
|
|
17
|
+
});
|
|
18
|
+
const encoded = payload?.payload?.data;
|
|
19
|
+
if (typeof encoded !== "string" || !encoded) {
|
|
20
|
+
throw new CiteArkError("Secret Manager 未返回密钥内容");
|
|
21
|
+
}
|
|
22
|
+
return Buffer.from(encoded, "base64").toString("utf8");
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function requiredSecretVersion(value) {
|
|
27
|
+
if (
|
|
28
|
+
typeof value !== "string"
|
|
29
|
+
|| !/^projects\/[^/]+\/secrets\/[^/]+\/versions\/[^/]+$/.test(value.trim())
|
|
30
|
+
) {
|
|
31
|
+
throw new TypeError("Secret Manager 版本必须是 projects/.../secrets/.../versions/... 格式");
|
|
32
|
+
}
|
|
33
|
+
return value.trim();
|
|
34
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import { googleApiError } from "./rest.mjs";
|
|
4
|
+
import { CiteArkError } from "../util.mjs";
|
|
5
|
+
|
|
6
|
+
const SIGNING_ALGORITHM = "GOOG4-RSA-SHA256";
|
|
7
|
+
const SIGNING_TERMINATOR = "goog4_request";
|
|
8
|
+
const MAXIMUM_EXPIRATION_SECONDS = 7 * 24 * 60 * 60;
|
|
9
|
+
|
|
10
|
+
export class GcpIamCredentialsSigner {
|
|
11
|
+
constructor({ auth, fetchImpl = fetch } = {}) {
|
|
12
|
+
if (!auth || typeof auth.headers !== "function") {
|
|
13
|
+
throw new TypeError("IAM Credentials signer 需要 Google 认证提供器");
|
|
14
|
+
}
|
|
15
|
+
this.auth = auth;
|
|
16
|
+
this.fetchImpl = fetchImpl;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async signBlob({ serviceAccount, payload }) {
|
|
20
|
+
const account = required(serviceAccount, "serviceAccount");
|
|
21
|
+
const bytes = Buffer.isBuffer(payload) ? payload : Buffer.from(String(payload), "utf8");
|
|
22
|
+
const url = new URL(
|
|
23
|
+
`https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${encodeURIComponent(account)}:signBlob`,
|
|
24
|
+
);
|
|
25
|
+
const response = await this.fetchImpl(url, {
|
|
26
|
+
method: "POST",
|
|
27
|
+
headers: {
|
|
28
|
+
...await this.auth.headers(),
|
|
29
|
+
"content-type": "application/json; charset=UTF-8",
|
|
30
|
+
},
|
|
31
|
+
body: JSON.stringify({ payload: bytes.toString("base64") }),
|
|
32
|
+
});
|
|
33
|
+
if (!response.ok) throw await googleApiError(response, "POST", url);
|
|
34
|
+
const signed = Buffer.from(String((await response.json())?.signedBlob ?? ""), "base64");
|
|
35
|
+
if (!signed.length) throw new CiteArkError("IAM Credentials signBlob 没有返回签名");
|
|
36
|
+
return signed;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export async function createGcsV4SignedReadUrl({
|
|
41
|
+
bucket,
|
|
42
|
+
object,
|
|
43
|
+
serviceAccount,
|
|
44
|
+
signer,
|
|
45
|
+
expiresSeconds = 30 * 60,
|
|
46
|
+
now = new Date(),
|
|
47
|
+
}) {
|
|
48
|
+
const normalizedBucket = required(bucket, "bucket");
|
|
49
|
+
const normalizedObject = required(object, "object");
|
|
50
|
+
const account = required(serviceAccount, "serviceAccount");
|
|
51
|
+
if (!signer || typeof signer.signBlob !== "function") {
|
|
52
|
+
throw new TypeError("GCS 签名 URL 需要 signBlob 实现");
|
|
53
|
+
}
|
|
54
|
+
const expiration = positiveInteger(expiresSeconds, "expiresSeconds");
|
|
55
|
+
if (expiration > MAXIMUM_EXPIRATION_SECONDS) {
|
|
56
|
+
throw new TypeError("GCS V4 签名 URL 有效期不能超过 7 天");
|
|
57
|
+
}
|
|
58
|
+
const instant = now instanceof Date ? now : new Date(now);
|
|
59
|
+
if (!Number.isFinite(instant.getTime())) throw new TypeError("GCS 签名时间无效");
|
|
60
|
+
|
|
61
|
+
const timestamp = instant.toISOString().replaceAll(/[-:]|\.\d{3}/g, "");
|
|
62
|
+
const date = timestamp.slice(0, 8);
|
|
63
|
+
const credentialScope = `${date}/auto/storage/${SIGNING_TERMINATOR}`;
|
|
64
|
+
const canonicalUri = `/${encodePathSegment(normalizedBucket)}/${normalizedObject
|
|
65
|
+
.split("/")
|
|
66
|
+
.map(encodePathSegment)
|
|
67
|
+
.join("/")}`;
|
|
68
|
+
const canonicalQuery = canonicalQueryString({
|
|
69
|
+
"X-Goog-Algorithm": SIGNING_ALGORITHM,
|
|
70
|
+
"X-Goog-Credential": `${account}/${credentialScope}`,
|
|
71
|
+
"X-Goog-Date": timestamp,
|
|
72
|
+
"X-Goog-Expires": String(expiration),
|
|
73
|
+
"X-Goog-SignedHeaders": "host",
|
|
74
|
+
});
|
|
75
|
+
const canonicalRequest = [
|
|
76
|
+
"GET",
|
|
77
|
+
canonicalUri,
|
|
78
|
+
canonicalQuery,
|
|
79
|
+
"host:storage.googleapis.com\n",
|
|
80
|
+
"host",
|
|
81
|
+
"UNSIGNED-PAYLOAD",
|
|
82
|
+
].join("\n");
|
|
83
|
+
const stringToSign = [
|
|
84
|
+
SIGNING_ALGORITHM,
|
|
85
|
+
timestamp,
|
|
86
|
+
credentialScope,
|
|
87
|
+
createHash("sha256").update(canonicalRequest).digest("hex"),
|
|
88
|
+
].join("\n");
|
|
89
|
+
const signature = await signer.signBlob({
|
|
90
|
+
serviceAccount: account,
|
|
91
|
+
payload: Buffer.from(stringToSign, "utf8"),
|
|
92
|
+
});
|
|
93
|
+
return `https://storage.googleapis.com${canonicalUri}?${canonicalQuery}&X-Goog-Signature=${signature.toString("hex")}`;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function canonicalQueryString(values) {
|
|
97
|
+
return Object.entries(values)
|
|
98
|
+
.map(([key, value]) => [strictEncode(key), strictEncode(value)])
|
|
99
|
+
.sort(([leftKey, leftValue], [rightKey, rightValue]) => (
|
|
100
|
+
compareAscii(leftKey, rightKey) || compareAscii(leftValue, rightValue)
|
|
101
|
+
))
|
|
102
|
+
.map(([key, value]) => `${key}=${value}`)
|
|
103
|
+
.join("&");
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function compareAscii(left, right) {
|
|
107
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function strictEncode(value) {
|
|
111
|
+
return encodeURIComponent(String(value)).replaceAll(/[!'()*]/g, (character) => (
|
|
112
|
+
`%${character.charCodeAt(0).toString(16).toUpperCase()}`
|
|
113
|
+
));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function encodePathSegment(value) {
|
|
117
|
+
return strictEncode(value);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function required(value, field) {
|
|
121
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
122
|
+
throw new TypeError(`GCS 签名 URL 缺少 ${field}`);
|
|
123
|
+
}
|
|
124
|
+
return value.trim();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function positiveInteger(value, field) {
|
|
128
|
+
const numeric = Number(value);
|
|
129
|
+
if (!Number.isInteger(numeric) || numeric <= 0) {
|
|
130
|
+
throw new TypeError(`GCS 签名 URL ${field} 必须是正整数`);
|
|
131
|
+
}
|
|
132
|
+
return numeric;
|
|
133
|
+
}
|