@lmmslab/harness 0.0.0-stage → 0.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1356 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +232 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Packages: everything the harness can do beyond its core comes in a package, installed
|
|
2
|
+
from the registry at harness.lmms-lab.com (or another source) ahead of time or the first
|
|
3
|
+
time a run needs it.
|
|
4
|
+
|
|
5
|
+
from harness import packages
|
|
6
|
+
|
|
7
|
+
packages.install(["minecraft"]) # and what it depends on
|
|
8
|
+
packages.installed() # [{name, version, source, provides, ...}]
|
|
9
|
+
packages.search("robot"), packages.info("robodojo")
|
|
10
|
+
packages.update(), packages.remove(["minecraft"])
|
|
11
|
+
packages.ensure("embodiments", "minecraft") # install on first use (packages.auto_install)
|
|
12
|
+
packages.describe(packages.info("robodojo")) # its fields and lines as every frontend shows them
|
|
13
|
+
|
|
14
|
+
The same as `harness install / remove / update / list / search / info`, the app's
|
|
15
|
+
Packages page. The modules:
|
|
16
|
+
|
|
17
|
+
manifest.py package.toml, and what a package provides by convention
|
|
18
|
+
versions.py versions and ranges
|
|
19
|
+
sources.py registry names, git, archive URLs, folders
|
|
20
|
+
registry.py the registry's index and files; publishing a version
|
|
21
|
+
archive.py deterministic package files, unpacked safely
|
|
22
|
+
resolve.py one version of each package, or why not
|
|
23
|
+
state.py the store, the user's list, a project's harness.toml and harness.lock
|
|
24
|
+
manager.py install, remove, update, trust, install on first use
|
|
25
|
+
describe.py a package as the frontends show it: its row of a list, its lines
|
|
26
|
+
addons.py the skills packages hold (Harness skills and Agent Skills)
|
|
27
|
+
migrate.py the app's installed skills becoming packages
|
|
28
|
+
environments.py the environments services and benches run in
|
|
29
|
+
provision.py installing a run's services and benches (environments, weights, assets)
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from .describe import COLUMNS, describe, rows
|
|
33
|
+
from .manager import AUTO_INSTALL, MissingPackage, Packages, activate
|
|
34
|
+
from .manifest import PACKAGE_KEYS, PROVIDES, Package, load
|
|
35
|
+
from .registry import OFFICIAL, Registry, RegistryError
|
|
36
|
+
from .resolve import ResolveError
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def install(specs=(), **options) -> dict:
|
|
40
|
+
return Packages().install(specs, **options)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def remove(names, **options) -> list[str]:
|
|
44
|
+
return Packages().remove(names, **options)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def update(names=(), **options) -> dict:
|
|
48
|
+
return Packages().update(names, **options)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def installed() -> list[dict]:
|
|
52
|
+
return Packages().installed()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def enabled() -> list[Package]:
|
|
56
|
+
return Packages().enabled()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def search(query: str = "") -> list[dict]:
|
|
60
|
+
return Packages().search(query)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def info(name: str) -> dict:
|
|
64
|
+
return Packages().info(name)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def ensure(what: str, name: str, **options) -> Package:
|
|
68
|
+
return Packages().ensure(what, name, **options)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
__all__ = ["AUTO_INSTALL", "COLUMNS", "MissingPackage", "OFFICIAL", "PACKAGE_KEYS", "PROVIDES", "Package",
|
|
72
|
+
"Packages", "Registry", "RegistryError", "ResolveError", "activate", "describe", "enabled", "ensure",
|
|
73
|
+
"info", "install", "installed", "load", "remove", "rows", "search", "update"]
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Skills in packages, in the two formats a package can hold:
|
|
2
|
+
|
|
3
|
+
- Harness skills: skills/<name>/ with skill.toml and Python, run in the kernel like the
|
|
4
|
+
built-in ones (runtime/execution/skills.py). Its `requirements` are installed into the skill's
|
|
5
|
+
own .deps folder with the kernel's Python when the package is installed.
|
|
6
|
+
- Agent Skills: agent-skills/<name>/ with SKILL.md (name and description in its front
|
|
7
|
+
matter, then instructions), the format of Claude's and OpenAI's skills. The agent is
|
|
8
|
+
told each one's name and description (the prompt's installed section,
|
|
9
|
+
runtime/decision/prompts.py), and reads the rest with read_skill() when a task needs it
|
|
10
|
+
(skills/instructions).
|
|
11
|
+
|
|
12
|
+
A folder or archive with skills but no package.toml (what the app's "Add a skill" took)
|
|
13
|
+
becomes one package per skill (find_skills(), manager.py).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
import re
|
|
19
|
+
import tomllib
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
NAME = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def front_matter(text: str) -> dict:
|
|
26
|
+
"""The front matter of a SKILL.md (between --- lines): key: value lines, quotes
|
|
27
|
+
stripped, a `>` or `|` value taking the indented lines after it."""
|
|
28
|
+
lines = text.lstrip("").splitlines()
|
|
29
|
+
if not lines or lines[0].strip() != "---":
|
|
30
|
+
return {}
|
|
31
|
+
data, key, block = {}, None, None
|
|
32
|
+
for line in lines[1:]:
|
|
33
|
+
if line.strip() == "---":
|
|
34
|
+
break
|
|
35
|
+
if block is not None and (line.startswith((" ", "\t")) or not line.strip()):
|
|
36
|
+
block.append(line.strip())
|
|
37
|
+
data[key] = " ".join(part for part in block if part)
|
|
38
|
+
continue
|
|
39
|
+
block = None
|
|
40
|
+
match = re.match(r"^([A-Za-z0-9_-]+):\s*(.*)$", line)
|
|
41
|
+
if not match:
|
|
42
|
+
continue
|
|
43
|
+
key, value = match.group(1), match.group(2).strip()
|
|
44
|
+
if value in (">", "|", ">-", "|-"):
|
|
45
|
+
block = []
|
|
46
|
+
data[key] = ""
|
|
47
|
+
else:
|
|
48
|
+
data[key] = value.strip("\"'")
|
|
49
|
+
return data
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def agent_skill(folder: Path) -> dict:
|
|
53
|
+
"""An Agent Skill folder: {name, format, summary, code, requirements, folder}."""
|
|
54
|
+
text = (folder / "SKILL.md").read_text(encoding="utf-8", errors="replace")
|
|
55
|
+
meta = front_matter(text)
|
|
56
|
+
name = (meta.get("name") or folder.name).strip().lower()
|
|
57
|
+
if not NAME.match(name):
|
|
58
|
+
raise ValueError(f"{folder.name}: the skill's name {name!r} is not a short lowercase name")
|
|
59
|
+
if not meta.get("description"):
|
|
60
|
+
raise ValueError(f"{name}: SKILL.md has no description in its front matter")
|
|
61
|
+
return {"name": name, "format": "agent", "summary": meta["description"][:500], "code": False,
|
|
62
|
+
"requirements": [], "folder": folder}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def harness_skill(folder: Path) -> dict:
|
|
66
|
+
"""A Harness skill folder: {name, format, summary, code, requirements, embodiments, folder}."""
|
|
67
|
+
from harness.runtime.execution.skills import _skill
|
|
68
|
+
|
|
69
|
+
skill = _skill(folder / "skill.toml")
|
|
70
|
+
if not NAME.match(skill.name) or "_" in skill.name:
|
|
71
|
+
raise ValueError(f"{skill.name}: not a short lowercase name")
|
|
72
|
+
if not skill.mixin.startswith(f"skills.{skill.name}"):
|
|
73
|
+
raise ValueError(f"{skill.name}: its mixin must be in its own package (skills.{skill.name})")
|
|
74
|
+
with (folder / "skill.toml").open("rb") as handle:
|
|
75
|
+
requirements = list(tomllib.load(handle).get("requirements", []))
|
|
76
|
+
return {"name": skill.name, "format": "harness", "summary": skill.summary, "code": True,
|
|
77
|
+
"requirements": requirements, "embodiments": list(skill.embodiments), "folder": folder}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def find_skills(folder: Path) -> tuple[list[dict], list[str]]:
|
|
81
|
+
"""The skills in a folder and its subfolders (a folder with skill.toml or SKILL.md is
|
|
82
|
+
one skill), and the problems of the ones that do not check."""
|
|
83
|
+
found, problems = [], []
|
|
84
|
+
for current, directories, files in os.walk(folder):
|
|
85
|
+
directories[:] = sorted(name for name in directories if not name.startswith(".") and name != "__pycache__")
|
|
86
|
+
current = Path(current)
|
|
87
|
+
try:
|
|
88
|
+
if "skill.toml" in files:
|
|
89
|
+
found.append(harness_skill(current))
|
|
90
|
+
elif "SKILL.md" in files:
|
|
91
|
+
found.append(agent_skill(current))
|
|
92
|
+
else:
|
|
93
|
+
continue
|
|
94
|
+
except (ValueError, OSError, KeyError, tomllib.TOMLDecodeError) as error:
|
|
95
|
+
problems.append(str(error))
|
|
96
|
+
directories[:] = []
|
|
97
|
+
return found, problems
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def skills(packages) -> list[dict]:
|
|
101
|
+
"""The skills of the enabled packages: [{name, format, summary, code, requirements,
|
|
102
|
+
package, path}] (with a problem when one does not check)."""
|
|
103
|
+
rows = []
|
|
104
|
+
for package in packages.enabled():
|
|
105
|
+
for kind, check in (("skills", harness_skill), ("agent_skills", agent_skill)):
|
|
106
|
+
for name in package.provides.get(kind, ()):
|
|
107
|
+
path = package.folder(kind) / name
|
|
108
|
+
try:
|
|
109
|
+
row = {key: value for key, value in check(path).items() if key != "folder"}
|
|
110
|
+
except (ValueError, OSError, KeyError, tomllib.TOMLDecodeError) as error:
|
|
111
|
+
row = {"name": name, "format": "harness" if kind == "skills" else "agent", "summary": "",
|
|
112
|
+
"code": kind == "skills", "requirements": [], "problem": str(error)}
|
|
113
|
+
rows.append({**row, "package": package.name, "path": str(path)})
|
|
114
|
+
return rows
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def agent_skill_options(packages, names) -> dict:
|
|
118
|
+
"""The kernel's options for read_skill: {name: {"description", "path"}} of the
|
|
119
|
+
Agent Skills `names`."""
|
|
120
|
+
rows = {row["name"]: row for row in skills(packages) if row["format"] == "agent" and "problem" not in row}
|
|
121
|
+
return {name: {"description": rows[name]["summary"], "path": rows[name]["path"]} for name in names if name in rows}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Package files: a deterministic tar.gz of a package folder, and unpacking one safely.
|
|
2
|
+
|
|
3
|
+
The same folder always gives the same bytes (files sorted, times, owners and modes
|
|
4
|
+
normalised, gzip without a name or time), so a published file's sha256 can be checked
|
|
5
|
+
against a rebuild. Unpacking refuses links, absolute paths, paths that leave the
|
|
6
|
+
folder and archives larger than MAX_FILES files or MAX_BYTES.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import gzip
|
|
10
|
+
import hashlib
|
|
11
|
+
import io
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import tarfile
|
|
15
|
+
import zipfile
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
MAX_FILES, MAX_BYTES = 20000, 200 * 1024 * 1024
|
|
19
|
+
# Never part of a package file.
|
|
20
|
+
SKIP_DIRS = {".git", "__pycache__", ".deps", ".pytest_cache", ".mypy_cache", "node_modules", ".venv"}
|
|
21
|
+
SKIP_SUFFIXES = (".pyc", ".pyo")
|
|
22
|
+
SKIP_FILES = {".DS_Store", ".installed.json"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def files(folder: Path) -> list[Path]:
|
|
26
|
+
"""The files of a package folder that go into its package file, sorted."""
|
|
27
|
+
folder = Path(folder)
|
|
28
|
+
out = []
|
|
29
|
+
for current, directories, names in os.walk(folder):
|
|
30
|
+
directories[:] = sorted(name for name in directories if name not in SKIP_DIRS)
|
|
31
|
+
for name in sorted(names):
|
|
32
|
+
path = Path(current) / name
|
|
33
|
+
if name in SKIP_FILES or name.endswith(SKIP_SUFFIXES) or path.is_symlink():
|
|
34
|
+
continue
|
|
35
|
+
out.append(path)
|
|
36
|
+
return sorted(out, key=lambda path: path.relative_to(folder).as_posix())
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def build(folder: Path, out: Path, *, prefix: str | None = None) -> str:
|
|
40
|
+
"""Write `folder` as a deterministic tar.gz at `out`, its files under `prefix/`
|
|
41
|
+
(default the folder's name); returns its sha256."""
|
|
42
|
+
folder = Path(folder)
|
|
43
|
+
prefix = prefix or folder.name
|
|
44
|
+
raw = io.BytesIO()
|
|
45
|
+
with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as tar:
|
|
46
|
+
directories = set()
|
|
47
|
+
for path in files(folder):
|
|
48
|
+
relative = path.relative_to(folder).as_posix()
|
|
49
|
+
parents = relative.split("/")[:-1]
|
|
50
|
+
for depth in range(len(parents) + 1):
|
|
51
|
+
directory = "/".join([prefix, *parents[:depth]])
|
|
52
|
+
if directory not in directories:
|
|
53
|
+
directories.add(directory)
|
|
54
|
+
tar.addfile(_info(directory, tarfile.DIRTYPE, 0o755))
|
|
55
|
+
data = path.read_bytes()
|
|
56
|
+
executable = os.access(path, os.X_OK) and os.name != "nt"
|
|
57
|
+
tar.addfile(_info(f"{prefix}/{relative}", tarfile.REGTYPE, 0o755 if executable else 0o644, len(data)),
|
|
58
|
+
io.BytesIO(data))
|
|
59
|
+
compressed = io.BytesIO()
|
|
60
|
+
with gzip.GzipFile(filename="", mode="wb", fileobj=compressed, mtime=0, compresslevel=9) as handle:
|
|
61
|
+
handle.write(raw.getvalue())
|
|
62
|
+
Path(out).parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
Path(out).write_bytes(compressed.getvalue())
|
|
64
|
+
return hashlib.sha256(compressed.getvalue()).hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _info(name: str, kind, mode: int, size: int = 0) -> tarfile.TarInfo:
|
|
68
|
+
info = tarfile.TarInfo(name)
|
|
69
|
+
info.type, info.mode, info.size, info.mtime = kind, mode, size, 0
|
|
70
|
+
info.uid = info.gid = 0
|
|
71
|
+
info.uname = info.gname = ""
|
|
72
|
+
return info
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _inside(root: Path, name: str) -> Path:
|
|
76
|
+
target = (root / name).resolve()
|
|
77
|
+
if name.startswith(("/", "\\")) or ":" in name.split("/")[0] or (target != root and root not in target.parents):
|
|
78
|
+
raise ValueError(f"the archive has a path outside it: {name}")
|
|
79
|
+
return target
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def unpack(path_or_bytes, into: Path) -> Path:
|
|
83
|
+
"""Unpack a .tar.gz, .tgz, .tar or .zip (a path or its bytes) into `into`; returns the
|
|
84
|
+
folder that holds package.toml or the skills (the single top-level folder when there
|
|
85
|
+
is one, else `into`)."""
|
|
86
|
+
into = Path(into)
|
|
87
|
+
into.mkdir(parents=True, exist_ok=True)
|
|
88
|
+
root = into.resolve()
|
|
89
|
+
data = path_or_bytes if isinstance(path_or_bytes, (bytes, bytearray)) else Path(path_or_bytes).read_bytes()
|
|
90
|
+
if data[:4] == b"PK\x03\x04":
|
|
91
|
+
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
|
92
|
+
members = archive.infolist()
|
|
93
|
+
if len(members) > MAX_FILES or sum(member.file_size for member in members) > MAX_BYTES:
|
|
94
|
+
raise ValueError("the archive is too large for a package")
|
|
95
|
+
for member in members:
|
|
96
|
+
_inside(root, member.filename)
|
|
97
|
+
archive.extractall(root)
|
|
98
|
+
else:
|
|
99
|
+
try:
|
|
100
|
+
tar = tarfile.open(fileobj=io.BytesIO(data), mode="r:*")
|
|
101
|
+
except tarfile.TarError:
|
|
102
|
+
raise ValueError("not a .tar.gz, .tar or .zip file") from None
|
|
103
|
+
with tar:
|
|
104
|
+
members = tar.getmembers()
|
|
105
|
+
if len(members) > MAX_FILES or sum(member.size for member in members) > MAX_BYTES:
|
|
106
|
+
raise ValueError("the archive is too large for a package")
|
|
107
|
+
for member in members:
|
|
108
|
+
if not (member.isfile() or member.isdir()):
|
|
109
|
+
raise ValueError(f"the archive has a link or a device: {member.name}")
|
|
110
|
+
_inside(root, member.name)
|
|
111
|
+
for member in members:
|
|
112
|
+
target = _inside(root, member.name)
|
|
113
|
+
if member.isdir():
|
|
114
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
115
|
+
continue
|
|
116
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
117
|
+
with tar.extractfile(member) as source:
|
|
118
|
+
target.write_bytes(source.read())
|
|
119
|
+
if member.mode & 0o111 and os.name != "nt":
|
|
120
|
+
target.chmod(0o755)
|
|
121
|
+
entries = [path for path in root.iterdir() if not path.name.startswith(".") and path.name != "__MACOSX"]
|
|
122
|
+
return entries[0] if len(entries) == 1 and entries[0].is_dir() else root
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""One package as every frontend shows it: describe() of an installed row, a registry row,
|
|
2
|
+
info() or install()'s found gives its columns, its facts [{key, label, value}] and its lines."""
|
|
3
|
+
|
|
4
|
+
from ..core.format import clip
|
|
5
|
+
|
|
6
|
+
# The columns of a list of installed packages and of a list of the registry's.
|
|
7
|
+
COLUMNS = {"installed": ("name", "version", "scope", "on", "source", "provides", "problem"),
|
|
8
|
+
"available": ("name", "version", "installed", "provides", "summary")}
|
|
9
|
+
# The labels of the facts whose key does not say them.
|
|
10
|
+
LABELS = {"depends": "depends on", "services": "services it can start", "environment": "installs an environment for",
|
|
11
|
+
"skill": "installs for the skill", "code": "contains Python code", "platforms": "runs on"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def describe(found: dict, width: int = 0) -> dict:
|
|
15
|
+
"""`found` as text; `width` cuts the provides column."""
|
|
16
|
+
versions, installed, listed = found.get("versions") or [], found.get("installed"), "enabled" in found
|
|
17
|
+
version = found.get("version") or (versions[-1]["version"] if versions else "")
|
|
18
|
+
provides = "; ".join(f"{kind.replace('_', ' ')}: {', '.join(map(str, names))}"
|
|
19
|
+
for kind, names in (found.get("provides") or {}).items() if names)
|
|
20
|
+
facts = [{"key": key, "label": LABELS.get(key, key), "value": str(value)} for key, value in [
|
|
21
|
+
("installed", f"{installed['version']} ({installed.get('scope', 'user')}) from " # info()'s
|
|
22
|
+
f"{str(installed.get('source') or '').replace('registry:', 'registry ', 1)}: {installed.get('path', '')}"
|
|
23
|
+
if isinstance(installed, dict) else installed),
|
|
24
|
+
("problem", found.get("problem")), ("provides", provides),
|
|
25
|
+
("depends", ", ".join(f"{name} {spec}".rstrip() for name, spec in (found.get("depends") or {}).items())),
|
|
26
|
+
("services", ", ".join(found.get("services") or [])),
|
|
27
|
+
*(("environment", f"{environment['for']}: {', '.join(environment.get('requirements') or []) or '-'}")
|
|
28
|
+
for environment in found.get("environments") or []),
|
|
29
|
+
*(("skill", f"{skill}: {', '.join(wanted)}") for skill, wanted in (found.get("skill_requirements") or {}).items()),
|
|
30
|
+
("commands", ", ".join(found.get("commands") or [])), ("code", "yes" if found.get("code") else ""),
|
|
31
|
+
("platforms", ", ".join(found.get("platforms") or [])),
|
|
32
|
+
("versions", ", ".join(row["version"] + ("" if row.get("fits", True) else " (not for this harness)") for row in versions)),
|
|
33
|
+
("registry", versions and found.get("registry")),
|
|
34
|
+
("problem", found.get("registry_problem") and f"registry: {found['registry_problem']}")] if value]
|
|
35
|
+
title = f"{found['name']} {version}".rstrip() + (f": {found['summary']}" if found.get("summary") else "")
|
|
36
|
+
return {"name": found["name"], "version": version, "summary": found.get("summary") or "",
|
|
37
|
+
"scope": found.get("scope", "user") if listed else "", "on": ("on" if found.get("enabled") else "off") if listed else "",
|
|
38
|
+
"source": str(found.get("source") or "").replace("registry:", "registry ", 1) + (" (linked)" if found.get("link") else ""),
|
|
39
|
+
"installed": installed.get("version", "") if isinstance(installed, dict) else installed or "",
|
|
40
|
+
"provides": clip(provides, width) if width else provides, "problem": found.get("problem") or "",
|
|
41
|
+
"facts": facts, "lines": [title, *(f"{item['label']}: {item['value']}" for item in facts)]}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def rows(found: list[dict], columns: str = "installed", width: int = 0) -> list[dict]:
|
|
45
|
+
"""A list of packages as the rows of a table of COLUMNS[columns]."""
|
|
46
|
+
return [{column: described[column] for column in COLUMNS[columns]} for described in (describe(item, width) for item in found)]
|