@citeark/agent 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +128 -0
- package/data/dataset-source-registry.v1.json +300 -0
- package/dist/arkgraph/boot.js +6 -0
- package/dist/arkgraph/index.html +1 -0
- package/dist/arkgraph/viewer.css +1 -0
- package/dist/arkgraph/viewer.en.css +1 -0
- package/dist/arkgraph/viewer.en.js +49 -0
- package/dist/arkgraph/viewer.en.js.LEGAL.txt +56 -0
- package/dist/arkgraph/viewer.js +49 -0
- package/dist/arkgraph/viewer.js.LEGAL.txt +56 -0
- package/docker/claude-code/Dockerfile +97 -0
- package/docker/claude-code/codex-pro-relay.mjs +466 -0
- package/docker/claude-code/runtime-contract-check.mjs +79 -0
- package/docs/arkgraph-reading.md +79 -0
- package/docs/configuration.md +100 -0
- package/docs/integration.md +92 -0
- package/docs/maturity-plan.md +27 -0
- package/docs/npm-release.md +44 -0
- package/docs/paper-reading.md +40 -0
- package/docs/research-plan-granularity.md +27 -0
- package/docs/terminal.md +49 -0
- package/examples/toy-evaluation/compile-task.json +27 -0
- package/examples/toy-evaluation/paper.md +5 -0
- package/examples/toy-evaluation/repository/README.md +9 -0
- package/examples/toy-evaluation/repository/checkpoint.json +4 -0
- package/examples/toy-evaluation/repository/evaluate.py +17 -0
- package/examples/toy-evaluation/task.json +81 -0
- package/package.json +59 -0
- package/prompts/compile-research.md +58 -0
- package/prompts/execute-contract.md +72 -0
- package/prompts/execute-workspace-simple.md +51 -0
- package/prompts/execute-workspace.md +34 -0
- package/prompts/prepare-reproduction.md +82 -0
- package/prompts/repair-research.md +45 -0
- package/protocol/CAP.md +129 -0
- package/protocol/LICENSE +12 -0
- package/protocol/MAPPINGS.md +72 -0
- package/protocol/README.md +38 -0
- package/protocol/conformance-v2.0-alpha.1.json +36 -0
- package/protocol/examples/arkgraph/checkpoint-evaluation.json +309 -0
- package/protocol/examples/arkgraph/fixtures.mjs +49 -0
- package/protocol/examples/arkgraph/paper-free.json +291 -0
- package/protocol/examples/arkgraph/partial-failure.json +344 -0
- package/protocol/examples/arkgraph/training-evaluation.json +443 -0
- package/protocol/profiles/agent-trace.md +16 -0
- package/protocol/profiles/computational-run.md +16 -0
- package/protocol/profiles/core.md +15 -0
- package/protocol/profiles/public-bundle.md +18 -0
- package/protocol/profiles/reproduction.md +29 -0
- package/protocol/profiles/research-compilation.md +44 -0
- package/protocol/profiles/research-plan.md +39 -0
- package/protocol/profiles/restricted-evidence.md +15 -0
- package/runtime/bootstrap-autodl-runtime.sh +314 -0
- package/runtime/create-runtime-venv.sh +41 -0
- package/runtime/install-local-cpu-runtime.sh +23 -0
- package/runtime/install-scientific-runtime.sh +153 -0
- package/runtime/mineru/parse.py +62 -0
- package/runtime/mineru/requirements.txt +4 -0
- package/runtime/requirements-baseline.txt +38 -0
- package/schemas/cap/v2/activity.schema.json +47 -0
- package/schemas/cap/v2/agent.schema.json +32 -0
- package/schemas/cap/v2/assertion.schema.json +110 -0
- package/schemas/cap/v2/descriptor.schema.json +243 -0
- package/schemas/cap/v2/entity.schema.json +64 -0
- package/schemas/cap/v2/manifest.schema.json +67 -0
- package/schemas/cap/v2/relation.schema.json +82 -0
- package/schemas/compute-catalog.schema.json +63 -0
- package/schemas/compute-decision.schema.json +27 -0
- package/schemas/execution-contract.schema.json +1024 -0
- package/schemas/research-card.schema.json +30 -0
- package/schemas/research-inventory-draft.schema.json +366 -0
- package/schemas/research.schema.json +1044 -0
- package/schemas/result.schema.json +173 -0
- package/schemas/verification-policy.schema.json +47 -0
- package/schemas/verified-conclusion.schema.json +58 -0
- package/schemas/workspace-summary.schema.json +24 -0
- package/scripts/build-arkgraph-view.mjs +12 -0
- package/scripts/check-execution-feasibility.mjs +24 -0
- package/scripts/check-syntax.mjs +15 -0
- package/scripts/deterministic-asset-preparation.py +438 -0
- package/scripts/package-cap.mjs +23 -0
- package/scripts/package-local-agent.mjs +23 -0
- package/scripts/preview-arkgraph.mjs +25 -0
- package/scripts/replay-research-compiler-candidate.mjs +134 -0
- package/scripts/review-compiler-sources.mjs +44 -0
- package/scripts/run-asset-preparation.sh +17 -0
- package/scripts/run-research-plan.mjs +98 -0
- package/scripts/validate-asset-preparation.py +290 -0
- package/scripts/verify-local-runtime.mjs +57 -0
- package/scripts/verify-npm-package.mjs +57 -0
- package/src/adapters/paper2agent.mjs +107 -0
- package/src/assets/cache.mjs +159 -0
- package/src/assets/compute.mjs +98 -0
- package/src/assets/executor.mjs +145 -0
- package/src/assets/lifecycle.mjs +213 -0
- package/src/assets/manifest.mjs +242 -0
- package/src/assets/opportunistic-preparation.mjs +81 -0
- package/src/assets/plan.mjs +411 -0
- package/src/assets/prompts.mjs +29 -0
- package/src/assets/public-asset-probe.mjs +525 -0
- package/src/assets/qualification.mjs +119 -0
- package/src/assets/readiness.mjs +130 -0
- package/src/assets/reproduction-admission.mjs +355 -0
- package/src/assets/requirements.mjs +152 -0
- package/src/assets/source-grounding.mjs +341 -0
- package/src/assets/source-policy.mjs +118 -0
- package/src/autodl/client.mjs +260 -0
- package/src/autodl/ssh.mjs +380 -0
- package/src/autodl/tools.mjs +129 -0
- package/src/cap/redaction.mjs +38 -0
- package/src/cap/v2/archive.mjs +152 -0
- package/src/cap/v2/attestation.mjs +204 -0
- package/src/cap/v2/canonical-json.mjs +114 -0
- package/src/cap/v2/compilation-artifact.mjs +240 -0
- package/src/cap/v2/core.mjs +282 -0
- package/src/cap/v2/measurement-assessment-records.mjs +23 -0
- package/src/cap/v2/pipeline-artifact.mjs +922 -0
- package/src/cap/v2/read.mjs +41 -0
- package/src/cap/v2/reassessment-artifact.mjs +383 -0
- package/src/cap/v2/research-artifact.mjs +231 -0
- package/src/cap/v2/research-map-records.mjs +46 -0
- package/src/cap/v2/research-object-records.mjs +163 -0
- package/src/cap/v2/research-records.mjs +187 -0
- package/src/cap/v2/verify.mjs +642 -0
- package/src/cli.mjs +1146 -0
- package/src/compute/autodl-pro-compiler.mjs +347 -0
- package/src/compute/autodl-pro-executor.mjs +459 -0
- package/src/compute/autodl-pro-job.mjs +843 -0
- package/src/compute/autodl-pro-network.mjs +295 -0
- package/src/compute/autodl-pro-remote.mjs +810 -0
- package/src/compute/autodl-pro-staging.mjs +117 -0
- package/src/compute/campaign.mjs +110 -0
- package/src/compute/catalog.mjs +123 -0
- package/src/compute/checkpoint-protocol.mjs +154 -0
- package/src/compute/codex-account-lock.mjs +111 -0
- package/src/compute/codex-account-session.mjs +107 -0
- package/src/compute/compiler-profile.mjs +38 -0
- package/src/compute/compiler-router.mjs +23 -0
- package/src/compute/coordinator-recovery.mjs +210 -0
- package/src/compute/executor-router.mjs +29 -0
- package/src/compute/gcp-batch-compiler.mjs +685 -0
- package/src/compute/gcp-batch-executor.mjs +1215 -0
- package/src/compute/gcp-batch-failure.mjs +92 -0
- package/src/compute/gcp-batch-job.mjs +527 -0
- package/src/compute/gcp-batch-lifecycle.mjs +81 -0
- package/src/compute/gcp-checkpoint-worker.mjs +1633 -0
- package/src/compute/local-codex-compiler.mjs +52 -0
- package/src/compute/measurement-hardware.mjs +128 -0
- package/src/compute/remote-attempt.mjs +226 -0
- package/src/compute/requirements.mjs +124 -0
- package/src/compute/research-phases.mjs +48 -0
- package/src/compute/scheduler.mjs +452 -0
- package/src/compute/shared-workloads.mjs +26 -0
- package/src/compute/stage-archive.mjs +79 -0
- package/src/contracts/campaign-contract.mjs +52 -0
- package/src/contracts/execution-contract.mjs +819 -0
- package/src/contracts/execution-mode.mjs +19 -0
- package/src/contracts/execution-timeouts.mjs +45 -0
- package/src/contracts/execution-workload.mjs +68 -0
- package/src/contracts/preflight-schema.mjs +25 -0
- package/src/contracts/public-contract.mjs +63 -0
- package/src/contracts/subject-tags.mjs +31 -0
- package/src/dashboard/data.mjs +898 -0
- package/src/dashboard/server.mjs +79 -0
- package/src/dashboard/static/dashboard.css +366 -0
- package/src/dashboard/static/dashboard.js +560 -0
- package/src/dashboard/static/index.html +85 -0
- package/src/deployment/community-policy.mjs +9 -0
- package/src/deployment/environment.mjs +112 -0
- package/src/deployment/guided.mjs +98 -0
- package/src/deployment/handoff.mjs +102 -0
- package/src/deployment/local-contract.mjs +31 -0
- package/src/deployment/local.mjs +100 -0
- package/src/deployment/prepare.mjs +46 -0
- package/src/deployment/recipe.mjs +108 -0
- package/src/deployment/supplement.mjs +51 -0
- package/src/deployment/terminal.mjs +43 -0
- package/src/diagnosis/renderer.mjs +75 -0
- package/src/diagnosis/target-failure.mjs +46 -0
- package/src/evidence/parser-registry.mjs +54 -0
- package/src/evidence/parsers/fasttext-classification.mjs +82 -0
- package/src/evidence/parsers/json-scalar.mjs +96 -0
- package/src/evidence/parsers/simcse-senteval.mjs +104 -0
- package/src/evidence/parsers/starspace-classification.mjs +78 -0
- package/src/evidence/registry.mjs +147 -0
- package/src/execution/runner-audit.mjs +473 -0
- package/src/gcp/auth.mjs +106 -0
- package/src/gcp/batch-client.mjs +120 -0
- package/src/gcp/resource-discovery.mjs +177 -0
- package/src/gcp/rest.mjs +82 -0
- package/src/gcp/secret-manager.mjs +34 -0
- package/src/gcp/signed-url.mjs +133 -0
- package/src/gcp/storage.mjs +220 -0
- package/src/graph/command.mjs +41 -0
- package/src/graph/execution.mjs +97 -0
- package/src/graph/model.mjs +37 -0
- package/src/graph/presentation.mjs +110 -0
- package/src/graph/query.mjs +159 -0
- package/src/graph/research-relations.mjs +69 -0
- package/src/graph/source-page.mjs +12 -0
- package/src/graph/source-preview.mjs +34 -0
- package/src/graph/validate.mjs +76 -0
- package/src/job.mjs +496 -0
- package/src/network/autodl-routing-proxy.mjs +462 -0
- package/src/network/egress-proxy.mjs +158 -0
- package/src/observability/event-contract.mjs +230 -0
- package/src/observability/pipeline-monitor.mjs +166 -0
- package/src/pipeline/orchestrator.mjs +1281 -0
- package/src/pipeline/recovery-error.mjs +11 -0
- package/src/pipeline/replay.mjs +304 -0
- package/src/pipeline/shared-execution.mjs +115 -0
- package/src/pipeline/stage-checkpoint.mjs +86 -0
- package/src/pipeline/stage-recovery.mjs +101 -0
- package/src/pipeline/targets.mjs +110 -0
- package/src/process.mjs +143 -0
- package/src/protocol.mjs +312 -0
- package/src/provider/codex-account.mjs +44 -0
- package/src/provider/codex-completion.mjs +49 -0
- package/src/provider/completion.mjs +292 -0
- package/src/provider/model-client.mjs +44 -0
- package/src/provider/model-route.mjs +29 -0
- package/src/provider/openrouter-readiness.mjs +189 -0
- package/src/provider/reader-bridge.mjs +35 -0
- package/src/provider/relay.mjs +263 -0
- package/src/provider/runtime-auth.mjs +40 -0
- package/src/public/cap.d.mts +90 -0
- package/src/public/cap.mjs +12 -0
- package/src/public/contracts.d.mts +2 -0
- package/src/public/host.mjs +171 -0
- package/src/public/operations.d.mts +11 -0
- package/src/public/presentation.d.mts +4 -0
- package/src/records/views.mjs +26 -0
- package/src/remote/command.mjs +178 -0
- package/src/remote/ssh.mjs +59 -0
- package/src/repository-origin.mjs +81 -0
- package/src/reproduction/evidence-feedback.mjs +96 -0
- package/src/reproduction/incomplete-initialization.mjs +25 -0
- package/src/reproduction/lifecycle.mjs +253 -0
- package/src/reproduction/plan.mjs +132 -0
- package/src/reproduction/prompts.mjs +70 -0
- package/src/reproduction/runner.mjs +188 -0
- package/src/reproduction/summary.mjs +130 -0
- package/src/reproduction/workspace-mode.mjs +7 -0
- package/src/research/automatic-admission.mjs +156 -0
- package/src/research/compiler-coverage.mjs +85 -0
- package/src/research/compiler-failure.mjs +24 -0
- package/src/research/compiler-normalization-guards.mjs +112 -0
- package/src/research/compiler-repair.mjs +3 -0
- package/src/research/compiler.mjs +853 -0
- package/src/research/continuation-selection.mjs +26 -0
- package/src/research/execution-graph-context.mjs +43 -0
- package/src/research/experiment-importance.mjs +15 -0
- package/src/research/inventory-handoff.mjs +104 -0
- package/src/research/inventory-revisions.mjs +32 -0
- package/src/research/mineru-local.mjs +73 -0
- package/src/research/paper-command.mjs +19 -0
- package/src/research/paper-markdown.mjs +180 -0
- package/src/research/paper-source-map.mjs +69 -0
- package/src/research/planning-policy.mjs +88 -0
- package/src/research/reference-materials.mjs +11 -0
- package/src/research/reproduction-scope.mjs +30 -0
- package/src/research/research-map.mjs +94 -0
- package/src/research/research-objects.mjs +88 -0
- package/src/research/source-discovery.mjs +646 -0
- package/src/research/source-observations.mjs +75 -0
- package/src/research/source-review-cli-mcp.mjs +26 -0
- package/src/research/source-review-input.mjs +209 -0
- package/src/research/source-review-local-codex.mjs +36 -0
- package/src/research/source-review-model.mjs +70 -0
- package/src/research/source-review.mjs +173 -0
- package/src/research/structure.mjs +3163 -0
- package/src/research-card/renderer.mjs +277 -0
- package/src/research-card/verified-conclusion.mjs +143 -0
- package/src/results/output-registry.mjs +183 -0
- package/src/runtime/claude-code.mjs +52 -0
- package/src/runtime/codex-capacity-retry.mjs +87 -0
- package/src/runtime/codex.mjs +64 -0
- package/src/runtime/config.mjs +157 -0
- package/src/runtime/final-output.mjs +40 -0
- package/src/runtime/index.mjs +21 -0
- package/src/runtime/local-codex.mjs +74 -0
- package/src/runtime/opencode.mjs +95 -0
- package/src/runtime/prompt.mjs +13 -0
- package/src/sandbox/docker.mjs +363 -0
- package/src/settings/command.mjs +297 -0
- package/src/settings/store.mjs +119 -0
- package/src/telemetry/pricing.mjs +68 -0
- package/src/telemetry/usage.mjs +265 -0
- package/src/terminal/events.mjs +97 -0
- package/src/terminal/input.mjs +40 -0
- package/src/terminal/plain.mjs +40 -0
- package/src/terminal/remote-stream.mjs +22 -0
- package/src/terminal/screen.mjs +214 -0
- package/src/terminal/transcript.mjs +69 -0
- package/src/util.mjs +107 -0
- package/src/verification/ai-assessor.mjs +534 -0
- package/src/verification/claim-evaluator.mjs +242 -0
- package/src/verification/evidence-context.mjs +165 -0
- package/src/verification/evidence-reader.mjs +95 -0
- package/src/verification/integrity.mjs +570 -0
- package/src/verification/tolerance.mjs +32 -0
- package/src/workloads/cpu-research-preparation.mjs +56 -0
- package/src/workloads/definition.mjs +74 -0
- package/src/workloads/phase-aware-reproduction.mjs +46 -0
- package/src/workloads/reproduction.mjs +85 -0
- package/src/workspace/command.mjs +242 -0
- package/src/workspace/control.mjs +49 -0
- package/src/workspace/entry.mjs +28 -0
- package/src/workspace/input.mjs +93 -0
- package/src/workspace/interactive.mjs +94 -0
- package/src/workspace/jobs.mjs +418 -0
- package/src/workspace/session.mjs +97 -0
- package/src/workspace/worker.mjs +137 -0
- package/ui/arkgraph/ambient-motion.mjs +10 -0
- package/ui/arkgraph/app.jsx +153 -0
- package/ui/arkgraph/boot.js +6 -0
- package/ui/arkgraph/camera-motion.mjs +20 -0
- package/ui/arkgraph/context-reveal.mjs +39 -0
- package/ui/arkgraph/details.css +3 -0
- package/ui/arkgraph/entry.jsx +28 -0
- package/ui/arkgraph/experiment-curves.mjs +17 -0
- package/ui/arkgraph/experiment-selection.mjs +15 -0
- package/ui/arkgraph/experiment-style.css +26 -0
- package/ui/arkgraph/experiment-ui.jsx +32 -0
- package/ui/arkgraph/frame.html +1 -0
- package/ui/arkgraph/graph-gestures.mjs +62 -0
- package/ui/arkgraph/label-layout.mjs +57 -0
- package/ui/arkgraph/locales/en.json +229 -0
- package/ui/arkgraph/locales/source-types.json +15 -0
- package/ui/arkgraph/localization-build.mjs +27 -0
- package/ui/arkgraph/material-build.mjs +23 -0
- package/ui/arkgraph/material-colors.mjs +39 -0
- package/ui/arkgraph/material-style.css +15 -0
- package/ui/arkgraph/open-graph.jsx +326 -0
- package/ui/arkgraph/outline.jsx +49 -0
- package/ui/arkgraph/package-lock.json +888 -0
- package/ui/arkgraph/package.json +17 -0
- package/ui/arkgraph/reading-layout.mjs +130 -0
- package/ui/arkgraph/reading-presentation.mjs +73 -0
- package/ui/arkgraph/record-detail.css +51 -0
- package/ui/arkgraph/record-details.jsx +29 -0
- package/ui/arkgraph/research-types.mjs +31 -0
- package/ui/arkgraph/selection-mark.jsx +6 -0
- package/ui/arkgraph/soft-spine.mjs +26 -0
- package/ui/arkgraph/steering-style.css +187 -0
- package/ui/arkgraph/style.css +272 -0
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Prepare mechanically downloadable assets before asking a semantic Agent."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import datetime
|
|
7
|
+
import fnmatch
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path, PurePosixPath
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import tarfile
|
|
16
|
+
from urllib.parse import urlparse
|
|
17
|
+
import zipfile
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
FALLBACK_TO_AGENT = 86
|
|
21
|
+
JOB_ROOT = Path(os.environ.get("CITEARK_JOB_ROOT", "/job")).resolve()
|
|
22
|
+
PLAN_PATH = JOB_ROOT / "input" / "asset-preparation-plan.json"
|
|
23
|
+
MANIFEST_PATH = JOB_ROOT / "output" / "asset-preparation-manifest.json"
|
|
24
|
+
LOG_ROOT = JOB_ROOT / "output" / "logs" / "asset-preparation" / "deterministic"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def job_path(value: str) -> Path:
|
|
28
|
+
target = Path(value)
|
|
29
|
+
if target.is_absolute() and target.parts[:2] == ("/", "job"):
|
|
30
|
+
target = JOB_ROOT.joinpath(*target.parts[2:])
|
|
31
|
+
elif not target.is_absolute():
|
|
32
|
+
target = JOB_ROOT / target
|
|
33
|
+
resolved = target.resolve(strict=False)
|
|
34
|
+
if resolved != JOB_ROOT and JOB_ROOT not in resolved.parents:
|
|
35
|
+
raise ValueError(f"path escapes /job: {value}")
|
|
36
|
+
return resolved
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def digest_file(target: Path) -> str:
|
|
40
|
+
digest = hashlib.sha256()
|
|
41
|
+
with target.open("rb") as source:
|
|
42
|
+
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
|
43
|
+
digest.update(chunk)
|
|
44
|
+
return "sha256:" + digest.hexdigest()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def write_json(target: Path, value: object) -> None:
|
|
48
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
temporary = target.with_suffix(target.suffix + ".tmp")
|
|
50
|
+
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
51
|
+
temporary.replace(target)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def evidence_path(target: Path) -> str:
|
|
55
|
+
return target.relative_to(JOB_ROOT / "output").as_posix()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def safe_member(value: str) -> str | None:
|
|
59
|
+
normalized = value.replace("\\", "/").lstrip("./")
|
|
60
|
+
candidate = PurePosixPath(normalized)
|
|
61
|
+
if not normalized or candidate.is_absolute() or ".." in candidate.parts:
|
|
62
|
+
return None
|
|
63
|
+
return candidate.as_posix()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def selected_relative(member: str, patterns: list[str], common_prefix: str | None) -> str | None:
|
|
67
|
+
candidates = [member]
|
|
68
|
+
if common_prefix and member.startswith(common_prefix + "/"):
|
|
69
|
+
candidates.insert(0, member[len(common_prefix) + 1 :])
|
|
70
|
+
for candidate in candidates:
|
|
71
|
+
if not patterns or any(fnmatch.fnmatchcase(candidate, pattern) for pattern in patterns):
|
|
72
|
+
return candidate
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_archive(
|
|
77
|
+
archive: Path,
|
|
78
|
+
destination: Path,
|
|
79
|
+
required: list[str],
|
|
80
|
+
excluded: list[str],
|
|
81
|
+
maximum_bytes: int,
|
|
82
|
+
) -> list[str]:
|
|
83
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
84
|
+
members: list[tuple[str, object]] = []
|
|
85
|
+
archive_type = ""
|
|
86
|
+
if tarfile.is_tarfile(archive):
|
|
87
|
+
archive_type = "tar"
|
|
88
|
+
with tarfile.open(archive, "r:*") as handle:
|
|
89
|
+
members = [(safe_member(member.name), member) for member in handle.getmembers() if member.isfile()]
|
|
90
|
+
names = [name for name, _ in members if name]
|
|
91
|
+
prefix = common_prefix(names)
|
|
92
|
+
selected = [
|
|
93
|
+
member for name, member in members
|
|
94
|
+
if name
|
|
95
|
+
and selected_relative(name, required, prefix)
|
|
96
|
+
and not any(fnmatch.fnmatchcase(selected_relative(name, required, prefix), pattern) for pattern in excluded)
|
|
97
|
+
]
|
|
98
|
+
if sum(member.size for member in selected) > maximum_bytes:
|
|
99
|
+
raise ValueError("tar archive exceeds the approved expanded-size budget")
|
|
100
|
+
written = []
|
|
101
|
+
for name, member in members:
|
|
102
|
+
if not name:
|
|
103
|
+
continue
|
|
104
|
+
relative = selected_relative(name, required, prefix)
|
|
105
|
+
if not relative or any(fnmatch.fnmatchcase(relative, pattern) for pattern in excluded):
|
|
106
|
+
continue
|
|
107
|
+
target = destination.joinpath(*PurePosixPath(relative).parts)
|
|
108
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
109
|
+
source = handle.extractfile(member)
|
|
110
|
+
if source is None:
|
|
111
|
+
continue
|
|
112
|
+
with source, target.open("wb") as output:
|
|
113
|
+
shutil.copyfileobj(source, output)
|
|
114
|
+
written.append(relative)
|
|
115
|
+
elif zipfile.is_zipfile(archive):
|
|
116
|
+
archive_type = "zip"
|
|
117
|
+
with zipfile.ZipFile(archive) as handle:
|
|
118
|
+
names = [safe_member(item.filename) for item in handle.infolist() if not item.is_dir()]
|
|
119
|
+
names = [name for name in names if name]
|
|
120
|
+
prefix = common_prefix(names)
|
|
121
|
+
selected = [
|
|
122
|
+
item for item in handle.infolist()
|
|
123
|
+
if not item.is_dir()
|
|
124
|
+
and safe_member(item.filename)
|
|
125
|
+
and selected_relative(safe_member(item.filename), required, prefix)
|
|
126
|
+
and not any(
|
|
127
|
+
fnmatch.fnmatchcase(
|
|
128
|
+
selected_relative(safe_member(item.filename), required, prefix),
|
|
129
|
+
pattern,
|
|
130
|
+
)
|
|
131
|
+
for pattern in excluded
|
|
132
|
+
)
|
|
133
|
+
]
|
|
134
|
+
if sum(item.file_size for item in selected) > maximum_bytes:
|
|
135
|
+
raise ValueError("zip archive exceeds the approved expanded-size budget")
|
|
136
|
+
written = []
|
|
137
|
+
for item in handle.infolist():
|
|
138
|
+
name = safe_member(item.filename)
|
|
139
|
+
if item.is_dir() or not name:
|
|
140
|
+
continue
|
|
141
|
+
relative = selected_relative(name, required, prefix)
|
|
142
|
+
if not relative or any(fnmatch.fnmatchcase(relative, pattern) for pattern in excluded):
|
|
143
|
+
continue
|
|
144
|
+
target = destination.joinpath(*PurePosixPath(relative).parts)
|
|
145
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
146
|
+
with handle.open(item) as source, target.open("wb") as output:
|
|
147
|
+
shutil.copyfileobj(source, output)
|
|
148
|
+
written.append(relative)
|
|
149
|
+
else:
|
|
150
|
+
return []
|
|
151
|
+
return written
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def common_prefix(names: list[str]) -> str | None:
|
|
155
|
+
first = {PurePosixPath(name).parts[0] for name in names if len(PurePosixPath(name).parts) > 1}
|
|
156
|
+
return next(iter(first)) if len(first) == 1 else None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def reset_directory(target: Path) -> None:
|
|
160
|
+
if target.exists():
|
|
161
|
+
shutil.rmtree(target)
|
|
162
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def publish_content(source: Path, destination: Path) -> None:
|
|
166
|
+
reset_directory(destination)
|
|
167
|
+
for item in source.rglob("*"):
|
|
168
|
+
relative = item.relative_to(source)
|
|
169
|
+
target = destination / relative
|
|
170
|
+
if item.is_dir():
|
|
171
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
172
|
+
elif item.is_file():
|
|
173
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
174
|
+
shutil.copyfile(item, target)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def curl_download(url: str, destination: Path, log: Path, maximum_bytes: int) -> tuple[int, str]:
|
|
178
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
179
|
+
before = destination.stat().st_size if destination.exists() else 0
|
|
180
|
+
command = [
|
|
181
|
+
"curl", "--location", "--fail", "--retry", "5", "--retry-all-errors",
|
|
182
|
+
"--connect-timeout", "30", "--max-time", "0", "--continue-at", "-",
|
|
183
|
+
"--max-filesize", str(maximum_bytes),
|
|
184
|
+
"--output", str(destination), url,
|
|
185
|
+
]
|
|
186
|
+
result = subprocess.run(command, capture_output=True, text=True)
|
|
187
|
+
log.parent.mkdir(parents=True, exist_ok=True)
|
|
188
|
+
log.write_text(
|
|
189
|
+
json.dumps({
|
|
190
|
+
"url": url,
|
|
191
|
+
"exitCode": result.returncode,
|
|
192
|
+
"stderr": result.stderr[-8000:],
|
|
193
|
+
"resumedFromBytes": before,
|
|
194
|
+
"finishedAt": now(),
|
|
195
|
+
}, ensure_ascii=False, indent=2) + "\n",
|
|
196
|
+
encoding="utf-8",
|
|
197
|
+
)
|
|
198
|
+
if result.returncode != 0 or not destination.is_file():
|
|
199
|
+
raise RuntimeError(f"curl failed for {url}: exit {result.returncode}")
|
|
200
|
+
if destination.stat().st_size > maximum_bytes:
|
|
201
|
+
raise ValueError(f"download exceeds the approved {maximum_bytes}-byte budget")
|
|
202
|
+
return destination.stat().st_size, digest_file(destination)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def huggingface_identity(url: str) -> tuple[str, str] | None:
|
|
206
|
+
parsed = urlparse(url)
|
|
207
|
+
if parsed.hostname not in {"huggingface.co", "www.huggingface.co"}:
|
|
208
|
+
return None
|
|
209
|
+
parts = [part for part in parsed.path.split("/") if part]
|
|
210
|
+
if parts[:1] == ["datasets"] and len(parts) >= 3:
|
|
211
|
+
return "datasets", "/".join(parts[1:3])
|
|
212
|
+
if len(parts) >= 2:
|
|
213
|
+
return "models", "/".join(parts[:2])
|
|
214
|
+
return None
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def prepare_asset(asset: dict) -> dict:
|
|
218
|
+
action = asset["decision"]["action"]
|
|
219
|
+
if action in {"stream", "inspect", "blocked"}:
|
|
220
|
+
raise NotImplementedError(f"deterministic fast path does not handle {action}")
|
|
221
|
+
locators = asset.get("sourceLocators", [])
|
|
222
|
+
if not locators:
|
|
223
|
+
raise NotImplementedError("asset has no public locator")
|
|
224
|
+
local_root = job_path(asset["localPath"])
|
|
225
|
+
local_root.mkdir(parents=True, exist_ok=True)
|
|
226
|
+
asset_log = LOG_ROOT / asset["id"]
|
|
227
|
+
asset_log.mkdir(parents=True, exist_ok=True)
|
|
228
|
+
attempted: list[dict] = []
|
|
229
|
+
selected = None
|
|
230
|
+
downloaded = 0
|
|
231
|
+
source_digest = None
|
|
232
|
+
commands: list[str] = []
|
|
233
|
+
if action == "metadata":
|
|
234
|
+
for source in locators:
|
|
235
|
+
url = source["url"]
|
|
236
|
+
result = subprocess.run(
|
|
237
|
+
["curl", "--head", "--location", "--fail", "--retry", "3", "--retry-all-errors", "--connect-timeout", "30", url],
|
|
238
|
+
capture_output=True,
|
|
239
|
+
text=True,
|
|
240
|
+
)
|
|
241
|
+
if result.returncode == 0:
|
|
242
|
+
attempted.append({"url": url, "status": "succeeded", "finishedAt": now()})
|
|
243
|
+
selected = source
|
|
244
|
+
commands.append(f"curl --head --location {url}")
|
|
245
|
+
break
|
|
246
|
+
attempted.append({"url": url, "status": "failed", "reason": result.stderr[-500:], "finishedAt": now()})
|
|
247
|
+
if selected is None:
|
|
248
|
+
raise RuntimeError("all approved metadata sources failed")
|
|
249
|
+
audit = asset_log / "metadata.json"
|
|
250
|
+
write_json(audit, {"assetId": asset["id"], "attemptedSources": attempted, "finishedAt": now()})
|
|
251
|
+
return {
|
|
252
|
+
"assetId": asset["id"],
|
|
253
|
+
"identity": asset["identity"],
|
|
254
|
+
**({"revision": asset["revision"]} if asset.get("revision") else {}),
|
|
255
|
+
"sourceLocator": selected["url"],
|
|
256
|
+
"localPath": asset["localPath"],
|
|
257
|
+
"status": "metadata_verified",
|
|
258
|
+
"verification": {
|
|
259
|
+
"status": "passed",
|
|
260
|
+
"commands": commands,
|
|
261
|
+
"evidence": [evidence_path(audit)],
|
|
262
|
+
},
|
|
263
|
+
}
|
|
264
|
+
for source in locators:
|
|
265
|
+
url = source["url"]
|
|
266
|
+
try:
|
|
267
|
+
attempt_downloaded = 0
|
|
268
|
+
attempt_commands: list[str] = []
|
|
269
|
+
maximum_download = asset["maximumDownloadBytes"]
|
|
270
|
+
attempt_key = hashlib.sha256(url.encode()).hexdigest()[:16]
|
|
271
|
+
attempt_root = JOB_ROOT / "runtime-cache" / "deterministic-assets" / asset["id"] / attempt_key
|
|
272
|
+
content_root = attempt_root / "content"
|
|
273
|
+
hf = huggingface_identity(url)
|
|
274
|
+
required = asset.get("requiredPaths", [])
|
|
275
|
+
if hf:
|
|
276
|
+
if not required or any(any(token in pattern for token in "*?[") for pattern in required):
|
|
277
|
+
raise NotImplementedError("Hugging Face fast path requires explicit file paths")
|
|
278
|
+
namespace, repository = hf
|
|
279
|
+
revision = asset.get("revision") or "main"
|
|
280
|
+
file_digests = []
|
|
281
|
+
for index, relative in enumerate(required, start=1):
|
|
282
|
+
file_url = f"https://huggingface.co/{'datasets/' if namespace == 'datasets' else ''}{repository}/resolve/{revision}/{relative}?download=true"
|
|
283
|
+
target = content_root.joinpath(*PurePosixPath(relative).parts)
|
|
284
|
+
size, file_digest = curl_download(
|
|
285
|
+
file_url,
|
|
286
|
+
target,
|
|
287
|
+
asset_log / f"transfer-{index}.json",
|
|
288
|
+
maximum_download,
|
|
289
|
+
)
|
|
290
|
+
attempt_downloaded += size
|
|
291
|
+
if attempt_downloaded > maximum_download:
|
|
292
|
+
raise ValueError("selected Hugging Face files exceed the approved download budget")
|
|
293
|
+
file_digests.append(f"{relative}\0{file_digest}")
|
|
294
|
+
attempt_commands.append(f"curl --continue-at - {file_url}")
|
|
295
|
+
source_digest = "sha256:" + hashlib.sha256("\n".join(file_digests).encode()).hexdigest()
|
|
296
|
+
else:
|
|
297
|
+
filename = Path(urlparse(url).path).name or "download.bin"
|
|
298
|
+
archive = attempt_root / "download" / filename
|
|
299
|
+
attempt_downloaded, source_digest = curl_download(
|
|
300
|
+
url,
|
|
301
|
+
archive,
|
|
302
|
+
asset_log / "transfer.json",
|
|
303
|
+
maximum_download,
|
|
304
|
+
)
|
|
305
|
+
attempt_commands.append(f"curl --continue-at - {url}")
|
|
306
|
+
expected = source.get("expectedDigest")
|
|
307
|
+
if expected and source_digest != expected:
|
|
308
|
+
raise ValueError(f"source digest mismatch: expected {expected}, got {source_digest}")
|
|
309
|
+
reset_directory(content_root)
|
|
310
|
+
written = extract_archive(
|
|
311
|
+
archive,
|
|
312
|
+
content_root,
|
|
313
|
+
asset.get("requiredPaths", []),
|
|
314
|
+
asset.get("excludedPaths", []),
|
|
315
|
+
asset["maximumExpandedBytes"],
|
|
316
|
+
)
|
|
317
|
+
if not written:
|
|
318
|
+
target_name = asset.get("requiredPaths", [filename])[0]
|
|
319
|
+
if any(token in target_name for token in "*?["):
|
|
320
|
+
target_name = filename
|
|
321
|
+
target = content_root.joinpath(*PurePosixPath(target_name).parts)
|
|
322
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
323
|
+
shutil.copyfile(archive, target)
|
|
324
|
+
expected = source.get("expectedDigest")
|
|
325
|
+
if expected and source_digest != expected:
|
|
326
|
+
raise ValueError(f"source digest mismatch: expected {expected}, got {source_digest}")
|
|
327
|
+
expanded = sum(item.stat().st_size for item in content_root.rglob("*") if item.is_file())
|
|
328
|
+
if expanded > asset["maximumExpandedBytes"]:
|
|
329
|
+
raise ValueError("prepared content exceeds the approved expanded-size budget")
|
|
330
|
+
publish_content(content_root, local_root)
|
|
331
|
+
downloaded = attempt_downloaded
|
|
332
|
+
commands = attempt_commands
|
|
333
|
+
attempted.append({"url": url, "status": "succeeded", "finishedAt": now()})
|
|
334
|
+
selected = source
|
|
335
|
+
break
|
|
336
|
+
except NotImplementedError:
|
|
337
|
+
raise
|
|
338
|
+
except Exception as error:
|
|
339
|
+
attempted.append({
|
|
340
|
+
"url": url,
|
|
341
|
+
"status": "failed",
|
|
342
|
+
"reason": str(error)[:500],
|
|
343
|
+
"finishedAt": now(),
|
|
344
|
+
})
|
|
345
|
+
if selected is None:
|
|
346
|
+
raise RuntimeError("all approved sources failed")
|
|
347
|
+
audit = asset_log / "acquisition.json"
|
|
348
|
+
write_json(audit, {
|
|
349
|
+
"assetId": asset["id"],
|
|
350
|
+
"sourceLocator": selected["url"],
|
|
351
|
+
"sourceClass": selected.get("sourceClass"),
|
|
352
|
+
"sourceDigest": source_digest,
|
|
353
|
+
"downloadBytes": downloaded,
|
|
354
|
+
"attemptedSources": attempted,
|
|
355
|
+
"finishedAt": now(),
|
|
356
|
+
})
|
|
357
|
+
return {
|
|
358
|
+
"assetId": asset["id"],
|
|
359
|
+
"identity": asset["identity"],
|
|
360
|
+
**({"revision": asset["revision"]} if asset.get("revision") else {}),
|
|
361
|
+
"sourceLocator": selected["url"],
|
|
362
|
+
"localPath": asset["localPath"],
|
|
363
|
+
"status": "prepared" if action == "prepare" else "metadata_verified",
|
|
364
|
+
"verification": {
|
|
365
|
+
"status": "passed",
|
|
366
|
+
"commands": commands + [f"sha256sum prepared files for {asset['id']}"],
|
|
367
|
+
"evidence": [evidence_path(audit)],
|
|
368
|
+
},
|
|
369
|
+
**({
|
|
370
|
+
"acquisition": {
|
|
371
|
+
"downloadBytes": downloaded,
|
|
372
|
+
"sourceDigest": source_digest,
|
|
373
|
+
"attemptedSources": attempted,
|
|
374
|
+
"commands": commands,
|
|
375
|
+
"evidence": [evidence_path(audit)],
|
|
376
|
+
},
|
|
377
|
+
} if action == "prepare" else {}),
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def run_check(check: dict) -> dict:
|
|
382
|
+
command = check.get("command")
|
|
383
|
+
if not command:
|
|
384
|
+
raise NotImplementedError(f"check {check['id']} needs semantic planning")
|
|
385
|
+
log = LOG_ROOT / "checks" / f"{check['id']}.log"
|
|
386
|
+
log.parent.mkdir(parents=True, exist_ok=True)
|
|
387
|
+
with log.open("w", encoding="utf-8") as output:
|
|
388
|
+
result = subprocess.run(
|
|
389
|
+
["/bin/bash", "-lc", command],
|
|
390
|
+
cwd=JOB_ROOT / "workspace" / "repository",
|
|
391
|
+
stdout=output,
|
|
392
|
+
stderr=subprocess.STDOUT,
|
|
393
|
+
text=True,
|
|
394
|
+
)
|
|
395
|
+
if result.returncode != 0:
|
|
396
|
+
raise RuntimeError(f"check {check['id']} failed with exit {result.returncode}")
|
|
397
|
+
return {
|
|
398
|
+
"checkId": check["id"],
|
|
399
|
+
"kind": check["kind"],
|
|
400
|
+
"status": "passed",
|
|
401
|
+
"commands": [command],
|
|
402
|
+
"evidence": [evidence_path(log)],
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def now() -> str:
|
|
407
|
+
return datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def main() -> int:
|
|
411
|
+
plan = json.loads(PLAN_PATH.read_text(encoding="utf-8"))
|
|
412
|
+
LOG_ROOT.mkdir(parents=True, exist_ok=True)
|
|
413
|
+
try:
|
|
414
|
+
records = [prepare_asset(asset) for asset in plan.get("assets", [])]
|
|
415
|
+
checks = [run_check(check) for check in plan.get("checks", [])]
|
|
416
|
+
except (NotImplementedError, RuntimeError, ValueError, OSError) as error:
|
|
417
|
+
write_json(LOG_ROOT / "fallback.json", {
|
|
418
|
+
"status": "agent_required",
|
|
419
|
+
"reason": str(error)[:1000],
|
|
420
|
+
"recordedAt": now(),
|
|
421
|
+
})
|
|
422
|
+
print(f"deterministic asset preparation handed off to Agent: {error}", file=sys.stderr)
|
|
423
|
+
return FALLBACK_TO_AGENT
|
|
424
|
+
manifest = {
|
|
425
|
+
"schemaVersion": "0.1",
|
|
426
|
+
"kind": "citeark.asset-preparation-manifest",
|
|
427
|
+
"planDigest": plan["planDigest"],
|
|
428
|
+
"preparationMode": "deterministic",
|
|
429
|
+
"assets": records,
|
|
430
|
+
"checks": checks,
|
|
431
|
+
}
|
|
432
|
+
write_json(MANIFEST_PATH, manifest)
|
|
433
|
+
print(json.dumps({"status": "prepared", "mode": "deterministic"}))
|
|
434
|
+
return 0
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
if __name__ == "__main__":
|
|
438
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { build } from 'esbuild';
|
|
2
|
+
import { mkdir, mkdtemp, writeFile, rm, copyFile } from 'node:fs/promises';
|
|
3
|
+
import { execFileSync } from 'node:child_process';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import os from 'node:os';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
const root = fileURLToPath(new URL('..', import.meta.url));
|
|
8
|
+
const out = path.resolve(process.argv[2] ?? path.join(root, 'dist'));
|
|
9
|
+
const temporary = await mkdtemp(path.join(os.tmpdir(), 'citeark-cap-package-'));
|
|
10
|
+
try {
|
|
11
|
+
const directory = path.join(temporary, 'package');
|
|
12
|
+
await mkdir(directory);
|
|
13
|
+
await mkdir(out, { recursive: true });
|
|
14
|
+
await build({ entryPoints: [path.join(root, 'src/public/cap.mjs')], outfile: path.join(directory, 'index.mjs'),
|
|
15
|
+
bundle: true, platform: 'node', format: 'esm', target: 'node22' });
|
|
16
|
+
await copyFile(path.join(root, 'src/public/cap.d.mts'), path.join(directory, 'index.d.mts'));
|
|
17
|
+
await copyFile(path.join(root, 'LICENSE'), path.join(directory, 'LICENSE'));
|
|
18
|
+
await writeFile(path.join(directory, 'package.json'), JSON.stringify({ name: 'citeark-cap', version: '2.0.0-alpha.2',
|
|
19
|
+
type: 'module', license: 'Apache-2.0', engines: { node: '>=22' },
|
|
20
|
+
exports: { '.': { types: './index.d.mts', default: './index.mjs' }, './package.json': './package.json' } }, null, 2)+'\n');
|
|
21
|
+
execFileSync('tar', ['-czf', path.join(out, 'cap.tgz'), '-C', temporary, 'package']);
|
|
22
|
+
} finally { await rm(temporary, { recursive: true, force: true }); }
|
|
23
|
+
console.log(`CAP protocol package: ${path.join(out, 'cap.tgz')}`);
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { copyFile, mkdir, readFile, stat } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { promisify } from 'node:util';
|
|
6
|
+
|
|
7
|
+
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
8
|
+
const outputDirectory = path.resolve(process.argv[2] ?? path.join(packageRoot, 'dist'));
|
|
9
|
+
const manifest = JSON.parse(await readFile(path.join(packageRoot, 'package.json'), 'utf8'));
|
|
10
|
+
const entries = ['package.json', 'package-lock.json', 'README.md', 'LICENSE', ...manifest.files];
|
|
11
|
+
for (const entry of entries) await stat(path.join(packageRoot, entry));
|
|
12
|
+
await mkdir(outputDirectory, { recursive: true });
|
|
13
|
+
|
|
14
|
+
const run = promisify(execFile);
|
|
15
|
+
// Keep the extracted-source archive for existing manual installs.
|
|
16
|
+
await run('tar', ['--exclude=*/node_modules', '-czf', path.join(outputDirectory, 'citeark-agent.tgz'), '-C', packageRoot, ...entries]);
|
|
17
|
+
// Website downloads and registry publishing use the same npm-generated bytes.
|
|
18
|
+
if (!process.env.npm_execpath) throw Error('请通过 npm run package:local 构建安装包。');
|
|
19
|
+
const { stdout } = await run(process.execPath, [process.env.npm_execpath, 'pack', '--json', '--ignore-scripts',
|
|
20
|
+
'--pack-destination', outputDirectory], { cwd: packageRoot });
|
|
21
|
+
const [packed] = JSON.parse(stdout);
|
|
22
|
+
await copyFile(path.join(outputDirectory, packed.filename), path.join(outputDirectory, 'cli.tgz'));
|
|
23
|
+
console.log(`Agent ${manifest.version}: ${packed.filename} (${packed.entryCount} files), cli.tgz, citeark-agent.tgz`);
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import {attachLocalSourcePreviews} from '../src/graph/source-preview.mjs';
|
|
2
|
+
import {readFile,writeFile,mkdir,copyFile,cp,mkdtemp,rm} from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import {createHash} from 'node:crypto';
|
|
6
|
+
import {fileURLToPath} from 'node:url';
|
|
7
|
+
import {readCapArchive} from '../src/cap/v2/read.mjs';
|
|
8
|
+
import {createArkGraph} from '../src/graph/query.mjs';
|
|
9
|
+
import {buildArkGraphPresentation} from '../src/graph/presentation.mjs';
|
|
10
|
+
const [archive,output,paper]=process.argv.slice(2);
|
|
11
|
+
if(!archive||!output)throw Error('Usage: node scripts/preview-arkgraph.mjs <signed.cap> <output-directory> [paper.pdf]');
|
|
12
|
+
const root=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..'),dir=path.resolve(output),temp=await mkdtemp(path.join(os.tmpdir(),'arkgraph-preview-'));
|
|
13
|
+
try{
|
|
14
|
+
const cap=await readCapArchive(path.resolve(archive),path.join(temp,'read'));
|
|
15
|
+
const view=buildArkGraphPresentation(createArkGraph([cap]));
|
|
16
|
+
await mkdir(dir,{recursive:true});
|
|
17
|
+
await cp(path.join(root,'dist/arkgraph'),dir,{recursive:true});
|
|
18
|
+
if(paper)await attachLocalSourcePreviews(view,path.resolve(paper),dir);
|
|
19
|
+
await writeFile(path.join(dir,'fold-data.json'),JSON.stringify(view));
|
|
20
|
+
const json=JSON.stringify(view).replaceAll('<','\\u003c');
|
|
21
|
+
const assetVersion=async name=>createHash('sha256').update(await readFile(path.join(dir,name))).digest('hex').slice(0,16);
|
|
22
|
+
const jsVersion=await assetVersion('viewer.js'),cssVersion=await assetVersion('viewer.css');
|
|
23
|
+
await writeFile(path.join(dir,'index.html'),`<!doctype html><html lang="zh-CN"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>ArkGraph</title><link rel="stylesheet" href="viewer.css?v=${cssVersion}"></head><body><div id="root"></div><script id="graphData" type="application/json">${json}</script><script src="viewer.js?v=${jsVersion}"></script></body></html>`);
|
|
24
|
+
console.log(JSON.stringify({output:dir,artifactDigest:view.digest,nodes:view.nodes.length,relations:view.edges.length}));
|
|
25
|
+
}finally{await rm(temp,{recursive:true,force:true});}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
attachCompilerAdvisories,
|
|
7
|
+
reviewCompilerCandidate,
|
|
8
|
+
} from "../src/research/compiler.mjs";
|
|
9
|
+
import {
|
|
10
|
+
compilerReconstructionRetryIssues,
|
|
11
|
+
normalizeCompilerResearchDraft,
|
|
12
|
+
} from "../src/research/structure.mjs";
|
|
13
|
+
import { CiteArkError, pathExists, readJson, writeJson } from "../src/util.mjs";
|
|
14
|
+
|
|
15
|
+
const options = parseOptions(process.argv.slice(2));
|
|
16
|
+
const runDirectory = path.resolve(options.run);
|
|
17
|
+
const outputPath = path.resolve(
|
|
18
|
+
options.output ?? path.join(runDirectory, "output", "research.replayed.normalized.json"),
|
|
19
|
+
);
|
|
20
|
+
const reportPath = path.resolve(
|
|
21
|
+
options.report ?? path.join(runDirectory, "execution", "deterministic-replay.json"),
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
const [draft, task, runState, repositoryIdentityInput, datasetRegistryInput, beforeSnapshot] = await Promise.all([
|
|
25
|
+
readJson(path.join(runDirectory, "output", "research.json")),
|
|
26
|
+
readJson(path.join(runDirectory, "input", "compile-task.json")),
|
|
27
|
+
readOptionalJson(path.join(runDirectory, "run.json")),
|
|
28
|
+
readOptionalJson(path.join(runDirectory, "input", "repository-identity.json")),
|
|
29
|
+
readOptionalJson(path.join(runDirectory, "input", "dataset-source-registry.json")),
|
|
30
|
+
readJson(path.join(runDirectory, "execution", "workspace-before.json")),
|
|
31
|
+
]);
|
|
32
|
+
const fixedPaperSource = (Array.isArray(draft?.sources) ? draft.sources : [])
|
|
33
|
+
.find((source) => source?.kind === "paper");
|
|
34
|
+
const repositoryIdentity = repositoryIdentityInput
|
|
35
|
+
?? runState?.repository?.identity
|
|
36
|
+
?? { available: false };
|
|
37
|
+
const datasetRegistry = datasetRegistryInput ?? { datasets: [] };
|
|
38
|
+
const repositorySnapshot = path.join(runDirectory, "snapshot", "repository");
|
|
39
|
+
const replayFallbacks = [
|
|
40
|
+
...(!repositoryIdentityInput && runState?.repository?.identity
|
|
41
|
+
? ["repository identity recovered from run.json"]
|
|
42
|
+
: []),
|
|
43
|
+
...(!datasetRegistryInput
|
|
44
|
+
? ["dataset source registry absent; asset-source findings may be conservative"]
|
|
45
|
+
: []),
|
|
46
|
+
...(!(await pathExists(repositorySnapshot))
|
|
47
|
+
? ["repository snapshot absent; repository-source findings may be conservative"]
|
|
48
|
+
: []),
|
|
49
|
+
];
|
|
50
|
+
const context = {
|
|
51
|
+
taskName: task.name,
|
|
52
|
+
computeContext: task.computeContext,
|
|
53
|
+
paperTitle: task.paper?.title ?? draft?.title,
|
|
54
|
+
paperUri: task.paper?.sourceUrl ?? task.paper?.resolvedUrl ?? fixedPaperSource?.uri,
|
|
55
|
+
paperDigest: task.paper?.digest ?? fixedPaperSource?.digest,
|
|
56
|
+
paperPath: containerInputPath(runDirectory, task.paper?.path),
|
|
57
|
+
repositoryRoot: repositorySnapshot,
|
|
58
|
+
repositoryIdentity,
|
|
59
|
+
repositoryFiles: (beforeSnapshot?.entries ?? [])
|
|
60
|
+
.filter((entry) => entry?.type === "file" && typeof entry.path === "string")
|
|
61
|
+
.map((entry) => entry.path),
|
|
62
|
+
licensePolicy: task.licensePolicy,
|
|
63
|
+
continuation: task.continuation ?? null,
|
|
64
|
+
};
|
|
65
|
+
const normalized = normalizeCompilerResearchDraft(draft, context);
|
|
66
|
+
const compilerIssues = compilerReconstructionRetryIssues(normalized.research, context);
|
|
67
|
+
const review = await reviewCompilerCandidate({
|
|
68
|
+
candidate: normalized.research,
|
|
69
|
+
draftContext: context,
|
|
70
|
+
reconstructionIssues: compilerIssues,
|
|
71
|
+
datasetRegistry,
|
|
72
|
+
});
|
|
73
|
+
const normalizationWarnings = attachCompilerAdvisories(
|
|
74
|
+
normalized.research,
|
|
75
|
+
normalized.warnings,
|
|
76
|
+
review.advisoryIssues,
|
|
77
|
+
);
|
|
78
|
+
const report = {
|
|
79
|
+
schemaVersion: "0.1",
|
|
80
|
+
runDirectory,
|
|
81
|
+
input: path.join(runDirectory, "output", "research.json"),
|
|
82
|
+
output: review.blockingIssues.length ? null : outputPath,
|
|
83
|
+
status: review.blockingIssues.length ? "invalid" : "valid_with_advisories",
|
|
84
|
+
agentPlanned: review.agentPlanned,
|
|
85
|
+
coverage: review.coverage,
|
|
86
|
+
replayFallbacks,
|
|
87
|
+
warnings: normalizationWarnings,
|
|
88
|
+
compilerIssues,
|
|
89
|
+
blockingIssues: review.blockingIssues,
|
|
90
|
+
advisoryIssues: review.advisoryIssues,
|
|
91
|
+
};
|
|
92
|
+
await writeJson(reportPath, report);
|
|
93
|
+
if (review.blockingIssues.length) {
|
|
94
|
+
throw new CiteArkError(`确定性回放仍有硬边界问题:\n- ${review.blockingIssues.join("\n- ")}`, { exitCode: 2 });
|
|
95
|
+
}
|
|
96
|
+
await writeJson(outputPath, normalized.research);
|
|
97
|
+
console.log("Deterministic replay: valid with advisories");
|
|
98
|
+
console.log(`Research: ${outputPath}`);
|
|
99
|
+
console.log(`Report: ${reportPath}`);
|
|
100
|
+
console.log(`Warnings: ${normalizationWarnings.length}`);
|
|
101
|
+
console.log(`Advisories: ${review.advisoryIssues.length}`);
|
|
102
|
+
|
|
103
|
+
function containerInputPath(runRoot, value) {
|
|
104
|
+
if (typeof value !== "string" || !value.startsWith("/job/input/")) {
|
|
105
|
+
throw new CiteArkError("编译任务没有固定的容器论文路径");
|
|
106
|
+
}
|
|
107
|
+
const relative = value.slice("/job/input/".length);
|
|
108
|
+
if (!relative || relative.split("/").includes("..")) {
|
|
109
|
+
throw new CiteArkError("编译任务论文路径越界");
|
|
110
|
+
}
|
|
111
|
+
return path.join(runRoot, "input", ...relative.split("/"));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function readOptionalJson(target) {
|
|
115
|
+
return await pathExists(target) ? readJson(target) : null;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function parseOptions(argv) {
|
|
119
|
+
const result = {};
|
|
120
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
121
|
+
const flag = argv[index];
|
|
122
|
+
if (!new Set(["--run", "--output", "--report"]).has(flag)) {
|
|
123
|
+
throw new CiteArkError(`未知参数:${flag}`);
|
|
124
|
+
}
|
|
125
|
+
const value = argv[index + 1];
|
|
126
|
+
if (!value || value.startsWith("--")) throw new CiteArkError(`${flag} 缺少值`);
|
|
127
|
+
result[flag.slice(2)] = value;
|
|
128
|
+
index += 1;
|
|
129
|
+
}
|
|
130
|
+
if (!result.run) {
|
|
131
|
+
throw new CiteArkError("用法:node scripts/replay-research-compiler-candidate.mjs --run <compiler-run-dir> [--output <research.json>] [--report <replay.json>]");
|
|
132
|
+
}
|
|
133
|
+
return result;
|
|
134
|
+
}
|