@h-ai/tools 0.0.0-stage → 1.0.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.en.md +37 -0
- package/README.ja.md +37 -0
- package/README.md +36 -2
- package/dist/chunk-UA7MOYTT.js +413 -0
- package/dist/chunk-UA7MOYTT.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/server/index.d.ts +306 -0
- package/dist/server/index.js +1051 -0
- package/dist/server/index.js.map +1 -0
- package/dist/tools-types-CM0lGuJr.d.ts +199 -0
- package/jobs/.dockerignore +4 -0
- package/jobs/README.md +223 -0
- package/jobs/archives.py +56 -0
- package/jobs/browser/Dockerfile +20 -0
- package/jobs/browser/operation.py +17 -0
- package/jobs/browser/render.mjs +90 -0
- package/jobs/browser_policy.py +36 -0
- package/jobs/documents/Dockerfile +23 -0
- package/jobs/documents/generate.py +165 -0
- package/jobs/documents/operation.py +84 -0
- package/jobs/documents/requirements.txt +3 -0
- package/jobs/hyperframes-runtime.mjs +73 -0
- package/jobs/media_files.py +78 -0
- package/jobs/ocr/Dockerfile +19 -0
- package/jobs/ocr/operation.py +18 -0
- package/jobs/office/Dockerfile +21 -0
- package/jobs/office/office_job.py +65 -0
- package/jobs/office/operation.py +43 -0
- package/jobs/preview_build/Dockerfile +19 -0
- package/jobs/preview_build/build.mjs +552 -0
- package/jobs/preview_build/operation.py +62 -0
- package/jobs/preview_build/package.json +11 -0
- package/jobs/tests/image-smoke.mjs +83 -0
- package/jobs/tests/requirements.txt +4 -0
- package/jobs/tests/test_documents_generate.py +107 -0
- package/jobs/tests/test_hai_tool.py +82 -0
- package/jobs/tests/test_media_jobs.py +449 -0
- package/jobs/tests/test_preview_build.py +213 -0
- package/jobs/tool.py +145 -0
- package/jobs/video_render/Dockerfile +20 -0
- package/jobs/video_render/operation.py +109 -0
- package/jobs/video_render/render.mjs +36 -0
- package/jobs/web_capture/Dockerfile +20 -0
- package/jobs/web_capture/capture.mjs +122 -0
- package/jobs/web_capture/operation.py +65 -0
- package/package.json +62 -3
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { mkdtemp, rename, rm, writeFile } from 'node:fs/promises'
|
|
2
|
+
import { dirname, join } from 'node:path'
|
|
3
|
+
import process from 'node:process'
|
|
4
|
+
import { setTimeout as delay } from 'node:timers/promises'
|
|
5
|
+
import { runHyperframes } from '../hyperframes-runtime.mjs'
|
|
6
|
+
|
|
7
|
+
const navigationFailures = []
|
|
8
|
+
|
|
9
|
+
function hasPhase(stderr, phase, status) {
|
|
10
|
+
return typeof stderr === 'string' && stderr.split('\n').some((line) => {
|
|
11
|
+
if (!line.startsWith('HYPERFRAMES_CAPTURE_PHASE '))
|
|
12
|
+
return false
|
|
13
|
+
try {
|
|
14
|
+
const entry = JSON.parse(line.slice('HYPERFRAMES_CAPTURE_PHASE '.length))
|
|
15
|
+
return entry.phase === phase && entry.status === status
|
|
16
|
+
}
|
|
17
|
+
catch { return false }
|
|
18
|
+
})
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function transientNavigation(cause, url) {
|
|
22
|
+
try {
|
|
23
|
+
const result = JSON.parse(cause.stdout)
|
|
24
|
+
return result.ok === false && result.error?.code === 'runtime_error'
|
|
25
|
+
&& result.error.message === `net::ERR_NETWORK_CHANGED at ${url}`
|
|
26
|
+
&& hasPhase(cause.stderr, 'browser', 'completed') && hasPhase(cause.stderr, 'navigation', 'started')
|
|
27
|
+
}
|
|
28
|
+
catch { return false }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function capture() {
|
|
32
|
+
const [proxy, url, output, maximum] = process.argv.slice(2)
|
|
33
|
+
// Python 外层为 180 秒,所有尝试、退避和发布共用更早的内部截止时间。
|
|
34
|
+
const deadline = performance.now() + 170_000
|
|
35
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
36
|
+
const remaining = Math.floor(deadline - performance.now())
|
|
37
|
+
if (remaining <= 0)
|
|
38
|
+
throw Object.assign(new Error('Capture deadline exceeded'), { code: 'CAPTURE_TIMEOUT' })
|
|
39
|
+
const temporary = await mkdtemp(join(dirname(output), '.capture-attempt-'))
|
|
40
|
+
const destination = join(temporary, 'capture')
|
|
41
|
+
try {
|
|
42
|
+
const { stdout } = await runHyperframes([
|
|
43
|
+
'capture',
|
|
44
|
+
url,
|
|
45
|
+
'--output',
|
|
46
|
+
destination,
|
|
47
|
+
'--max-screenshots',
|
|
48
|
+
maximum,
|
|
49
|
+
'--timeout',
|
|
50
|
+
String(Math.min(45_000, remaining)),
|
|
51
|
+
'--capture-budget',
|
|
52
|
+
String(Math.min(60_000, remaining)),
|
|
53
|
+
'--skip-vision',
|
|
54
|
+
'--json',
|
|
55
|
+
], dirname(output), {
|
|
56
|
+
HYPERFRAMES_CAPTURE_PROXY: proxy,
|
|
57
|
+
HYPERFRAMES_CHROME_SANDBOX: '1',
|
|
58
|
+
HYPERFRAMES_CAPTURE_DEADLINE_MS: String(Math.min(120_000, remaining)),
|
|
59
|
+
}, remaining)
|
|
60
|
+
if (performance.now() >= deadline)
|
|
61
|
+
throw Object.assign(new Error('Capture deadline exceeded'), { code: 'CAPTURE_TIMEOUT' })
|
|
62
|
+
const result = JSON.parse(stdout)
|
|
63
|
+
if (result.ok !== true || !(result.httpStatus >= 200 && result.httpStatus < 300) || !(result.screenshots > 0))
|
|
64
|
+
throw new Error('Capture did not complete successfully')
|
|
65
|
+
// 仅记录固定错误阶段和重试次数,不保留用户 URL、正文或敏感 stderr。
|
|
66
|
+
await writeFile(join(destination, 'capture-status.json'), JSON.stringify({
|
|
67
|
+
httpStatus: result.httpStatus,
|
|
68
|
+
dropped: result.dropped ?? {},
|
|
69
|
+
warnings: result.warnings ?? [],
|
|
70
|
+
retries: navigationFailures,
|
|
71
|
+
}), 'utf8')
|
|
72
|
+
await rename(destination, output)
|
|
73
|
+
await rm(join(dirname(output), 'capture-error.json'), { force: true })
|
|
74
|
+
return
|
|
75
|
+
}
|
|
76
|
+
catch (cause) {
|
|
77
|
+
if (performance.now() >= deadline || cause?.code === 'CAPTURE_TIMEOUT')
|
|
78
|
+
throw Object.assign(new Error('Capture deadline exceeded'), { code: 'CAPTURE_TIMEOUT' })
|
|
79
|
+
if (attempt !== 0 || !transientNavigation(cause, url))
|
|
80
|
+
throw cause
|
|
81
|
+
navigationFailures.push({ phase: 'navigation', code: 'ERR_NETWORK_CHANGED', attempt: 1 })
|
|
82
|
+
await writeFile(join(dirname(output), 'capture-error.json'), JSON.stringify({ code: 'TRANSIENT_NAVIGATION', firstFailure: navigationFailures[0] }))
|
|
83
|
+
await delay(Math.min(250, Math.max(0, deadline - performance.now())))
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
// 每次尝试各有目录;成功仅原子发布完整目录,失败的 BLOCKED.md 不进入产物。
|
|
87
|
+
await rm(temporary, { recursive: true, force: true })
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
capture().catch(async (cause) => {
|
|
93
|
+
const diagnostics = [
|
|
94
|
+
cause instanceof Error
|
|
95
|
+
? cause.message
|
|
96
|
+
: '',
|
|
97
|
+
typeof cause?.stdout === 'string'
|
|
98
|
+
? cause.stdout
|
|
99
|
+
: '',
|
|
100
|
+
typeof cause?.stderr === 'string'
|
|
101
|
+
? cause.stderr
|
|
102
|
+
: '',
|
|
103
|
+
].join('\n')
|
|
104
|
+
const browserLaunchCompleted = hasPhase(cause?.stderr, 'browser', 'completed')
|
|
105
|
+
let code
|
|
106
|
+
if (!browserLaunchCompleted && /No usable sandbox|Running as root without --no-sandbox|Failed to move to new namespace|SUID sandbox|sys_chroot\(|Zygote process exited prematurely|sandbox.*(?:not permitted|unavailable)/i.test(diagnostics))
|
|
107
|
+
code = 'SANDBOX_UNAVAILABLE'
|
|
108
|
+
else if (cause?.code === 'CAPTURE_TIMEOUT')
|
|
109
|
+
code = 'TIMEOUT'
|
|
110
|
+
else if (navigationFailures.length)
|
|
111
|
+
code = 'NAVIGATION_RETRY_FAILED'
|
|
112
|
+
if (code) {
|
|
113
|
+
await writeFile(join(dirname(process.argv[4]), 'capture-error.json'), JSON.stringify({
|
|
114
|
+
code,
|
|
115
|
+
...(navigationFailures.length
|
|
116
|
+
? { firstFailure: navigationFailures[0] }
|
|
117
|
+
: {}),
|
|
118
|
+
})).catch(() => {})
|
|
119
|
+
}
|
|
120
|
+
process.stderr.write('HyperFrames web.capture failed\n')
|
|
121
|
+
process.exitCode = 1
|
|
122
|
+
})
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""受控网页采集作业。"""
|
|
2
|
+
import json
|
|
3
|
+
import pathlib
|
|
4
|
+
from tool import TEXT_LIMIT, ToolError, publish, run_command
|
|
5
|
+
from browser_policy import deployment_proxy, parse_url
|
|
6
|
+
from media_files import CAPTURE_INSTRUCTION_FILES, bundle_capture, collect_files
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def capture(request, _source, artifact, workspace):
|
|
10
|
+
url = parse_url(request['url'])
|
|
11
|
+
maximum = request.get('maxScreenshots', 4)
|
|
12
|
+
if type(maximum) is not int or not 1 <= maximum <= 8 or artifact.suffix.lower() != '.zip':
|
|
13
|
+
raise ToolError('INVALID_REQUEST', 'Capture requires ZIP output and one to eight screenshots')
|
|
14
|
+
proxy = deployment_proxy()
|
|
15
|
+
worker = pathlib.Path(__file__).with_name('capture.mjs')
|
|
16
|
+
try:
|
|
17
|
+
run_command(['node', str(worker), proxy, url, str(workspace / 'capture'), str(maximum)], 180, workspace)
|
|
18
|
+
except ToolError:
|
|
19
|
+
diagnostic = workspace / 'capture-error.json'
|
|
20
|
+
code = None
|
|
21
|
+
try:
|
|
22
|
+
if diagnostic.is_file() and not diagnostic.is_symlink() and diagnostic.stat().st_size <= 1024:
|
|
23
|
+
value = json.loads(diagnostic.read_text(encoding='utf-8'))
|
|
24
|
+
code = value.get('code') if isinstance(value, dict) else None
|
|
25
|
+
except (OSError, ValueError, UnicodeError):
|
|
26
|
+
pass
|
|
27
|
+
if code == 'SANDBOX_UNAVAILABLE':
|
|
28
|
+
raise ToolError('SANDBOX_UNAVAILABLE', 'The environment cannot enable the required Chromium sandbox') from None
|
|
29
|
+
if code == 'TIMEOUT':
|
|
30
|
+
raise ToolError('TIMEOUT', 'The capture deadline was exceeded') from None
|
|
31
|
+
if code == 'NAVIGATION_RETRY_FAILED':
|
|
32
|
+
raise ToolError('NAVIGATION_RETRY_FAILED', 'Navigation failed after one transient network retry') from None
|
|
33
|
+
raise
|
|
34
|
+
directory = workspace / 'capture'
|
|
35
|
+
files = collect_files(directory, CAPTURE_INSTRUCTION_FILES)
|
|
36
|
+
names = [name for name, _ in files]
|
|
37
|
+
if 'extracted/tokens.json' not in names or 'extracted/visible-text.txt' not in names:
|
|
38
|
+
raise ToolError('INVALID_OUTPUT', 'Capture did not produce source text and design tokens')
|
|
39
|
+
tokens = json.loads((directory / 'extracted' / 'tokens.json').read_text(encoding='utf-8'))
|
|
40
|
+
if not isinstance(tokens, dict):
|
|
41
|
+
raise ToolError('INVALID_OUTPUT', 'Capture tokens are invalid')
|
|
42
|
+
colors = tokens.get('colors', [])
|
|
43
|
+
if not isinstance(colors, list) or any(not isinstance(color, str) or len(color) > 256 for color in colors):
|
|
44
|
+
raise ToolError('INVALID_OUTPUT', 'Capture colors are invalid')
|
|
45
|
+
text = (directory / 'extracted' / 'visible-text.txt').read_text(encoding='utf-8')
|
|
46
|
+
screenshots = [name for name in names if name.startswith('screenshots/') and name.endswith('.png')]
|
|
47
|
+
if not text.strip() or len(text.encode('utf-8')) > TEXT_LIMIT or not screenshots:
|
|
48
|
+
raise ToolError('INVALID_OUTPUT', 'Capture did not produce bounded text and screenshots')
|
|
49
|
+
summary = {
|
|
50
|
+
'sourceUrl': url,
|
|
51
|
+
'title': str(tokens.get('title', ''))[:2048],
|
|
52
|
+
'description': str(tokens.get('description', ''))[:8192],
|
|
53
|
+
'text': text,
|
|
54
|
+
'colors': colors[:20],
|
|
55
|
+
'screenshots': screenshots,
|
|
56
|
+
'materials': names,
|
|
57
|
+
}
|
|
58
|
+
document = {**summary, 'renderer': {'name': 'hyperframes', 'version': '0.9.56'}}
|
|
59
|
+
(directory / 'capture.json').write_text(json.dumps(document, ensure_ascii=False), encoding='utf-8')
|
|
60
|
+
result = workspace / 'capture.zip'
|
|
61
|
+
bundle_capture(directory, result)
|
|
62
|
+
return {**publish(result, artifact, b'PK\x03\x04'), 'summary': summary}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
commands = {'web.capture': ({'url', 'output'}, {'maxScreenshots'}, capture)}
|
package/package.json
CHANGED
|
@@ -1,6 +1,65 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@h-ai/tools",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"description": "
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "1.0.0-beta.31",
|
|
5
|
+
"description": "Model-neutral tools, validation, authorization and execution for hai-framework",
|
|
6
|
+
"license": "Apache-2.0",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"browser": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"default": "./dist/index.js"
|
|
12
|
+
},
|
|
13
|
+
"types": "./dist/index.d.ts",
|
|
14
|
+
"import": "./dist/index.js"
|
|
15
|
+
},
|
|
16
|
+
"./server": {
|
|
17
|
+
"types": "./dist/server/index.d.ts",
|
|
18
|
+
"import": "./dist/server/index.js"
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"main": "./dist/index.js",
|
|
22
|
+
"types": "./dist/index.d.ts",
|
|
23
|
+
"files": [
|
|
24
|
+
"dist",
|
|
25
|
+
"jobs"
|
|
26
|
+
],
|
|
27
|
+
"publishConfig": {
|
|
28
|
+
"access": "public"
|
|
29
|
+
},
|
|
30
|
+
"engines": {
|
|
31
|
+
"node": "^22.23.4 || ^24.22.1 || ^26.12.1"
|
|
32
|
+
},
|
|
33
|
+
"peerDependencies": {
|
|
34
|
+
"@h-ai/runner": "1.0.0-beta.31"
|
|
35
|
+
},
|
|
36
|
+
"peerDependenciesMeta": {
|
|
37
|
+
"@h-ai/runner": {
|
|
38
|
+
"optional": true
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"dependencies": {
|
|
42
|
+
"@h-ai/core": "1.0.0-beta.31",
|
|
43
|
+
"zod": "^4.6.7"
|
|
44
|
+
},
|
|
45
|
+
"devDependencies": {
|
|
46
|
+
"@h-ai/reldb": "1.0.0-beta.31",
|
|
47
|
+
"@h-ai/runner": "1.0.0-beta.31",
|
|
48
|
+
"rimraf": "^6.1.4",
|
|
49
|
+
"tsup": "^9.0.0",
|
|
50
|
+
"typescript": "^6.0.3",
|
|
51
|
+
"vitest": "^5.1.0"
|
|
52
|
+
},
|
|
53
|
+
"scripts": {
|
|
54
|
+
"build": "tsup",
|
|
55
|
+
"dev": "tsup --watch",
|
|
56
|
+
"typecheck": "node ../../scripts/typecheck.mjs --noEmit",
|
|
57
|
+
"lint": "eslint .",
|
|
58
|
+
"lint:fix": "eslint . --fix",
|
|
59
|
+
"test": "vitest run",
|
|
60
|
+
"test:jobs": "python3 -B -m unittest discover -s jobs/tests",
|
|
61
|
+
"test:watch": "vitest watch",
|
|
62
|
+
"test:coverage": "vitest run --coverage",
|
|
63
|
+
"build:clean": "rimraf dist"
|
|
64
|
+
}
|
|
6
65
|
}
|