@h-ai/tools 0.0.0-stage → 1.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.en.md +37 -0
- package/README.ja.md +37 -0
- package/README.md +36 -2
- package/dist/chunk-UA7MOYTT.js +413 -0
- package/dist/chunk-UA7MOYTT.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/server/index.d.ts +306 -0
- package/dist/server/index.js +1051 -0
- package/dist/server/index.js.map +1 -0
- package/dist/tools-types-CM0lGuJr.d.ts +199 -0
- package/jobs/.dockerignore +4 -0
- package/jobs/README.md +223 -0
- package/jobs/archives.py +56 -0
- package/jobs/browser/Dockerfile +20 -0
- package/jobs/browser/operation.py +17 -0
- package/jobs/browser/render.mjs +90 -0
- package/jobs/browser_policy.py +36 -0
- package/jobs/documents/Dockerfile +23 -0
- package/jobs/documents/generate.py +165 -0
- package/jobs/documents/operation.py +84 -0
- package/jobs/documents/requirements.txt +3 -0
- package/jobs/hyperframes-runtime.mjs +73 -0
- package/jobs/media_files.py +78 -0
- package/jobs/ocr/Dockerfile +19 -0
- package/jobs/ocr/operation.py +18 -0
- package/jobs/office/Dockerfile +21 -0
- package/jobs/office/office_job.py +65 -0
- package/jobs/office/operation.py +43 -0
- package/jobs/preview_build/Dockerfile +19 -0
- package/jobs/preview_build/build.mjs +552 -0
- package/jobs/preview_build/operation.py +62 -0
- package/jobs/preview_build/package.json +11 -0
- package/jobs/tests/image-smoke.mjs +83 -0
- package/jobs/tests/requirements.txt +4 -0
- package/jobs/tests/test_documents_generate.py +107 -0
- package/jobs/tests/test_hai_tool.py +82 -0
- package/jobs/tests/test_media_jobs.py +449 -0
- package/jobs/tests/test_preview_build.py +213 -0
- package/jobs/tool.py +145 -0
- package/jobs/video_render/Dockerfile +20 -0
- package/jobs/video_render/operation.py +109 -0
- package/jobs/video_render/render.mjs +36 -0
- package/jobs/web_capture/Dockerfile +20 -0
- package/jobs/web_capture/capture.mjs +122 -0
- package/jobs/web_capture/operation.py +65 -0
- package/package.json +62 -3
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
import io
|
|
3
|
+
import json
|
|
4
|
+
import pathlib
|
|
5
|
+
import stat
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
import unittest
|
|
10
|
+
import zipfile
|
|
11
|
+
from html.parser import HTMLParser
|
|
12
|
+
|
|
13
|
+
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
14
|
+
sys.path.insert(0, str(ROOT))
|
|
15
|
+
from tool import serve
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def load():
|
|
19
|
+
spec = importlib.util.spec_from_file_location('hai_preview_build', ROOT / 'preview_build' / 'operation.py')
|
|
20
|
+
module = importlib.util.module_from_spec(spec)
|
|
21
|
+
spec.loader.exec_module(module)
|
|
22
|
+
return module
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Scripts(HTMLParser):
|
|
26
|
+
def __init__(self):
|
|
27
|
+
super().__init__()
|
|
28
|
+
self.scripts = []
|
|
29
|
+
self.current = None
|
|
30
|
+
|
|
31
|
+
def handle_starttag(self, tag, attrs):
|
|
32
|
+
if tag == 'script':
|
|
33
|
+
self.current = ''
|
|
34
|
+
|
|
35
|
+
def handle_data(self, text):
|
|
36
|
+
if self.current is not None:
|
|
37
|
+
self.current += text
|
|
38
|
+
|
|
39
|
+
def handle_endtag(self, tag):
|
|
40
|
+
if tag == 'script' and self.current is not None:
|
|
41
|
+
self.scripts.append(self.current)
|
|
42
|
+
self.current = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class PreviewBuildTests(unittest.TestCase):
|
|
46
|
+
def setUp(self):
|
|
47
|
+
self.temp = tempfile.TemporaryDirectory()
|
|
48
|
+
self.addCleanup(self.temp.cleanup)
|
|
49
|
+
self.root = pathlib.Path(self.temp.name)
|
|
50
|
+
self.inputs = self.root / 'input'
|
|
51
|
+
self.outputs = self.root / 'output'
|
|
52
|
+
self.inputs.mkdir()
|
|
53
|
+
self.outputs.mkdir()
|
|
54
|
+
|
|
55
|
+
def invoke(self, entries, output='preview.html'):
|
|
56
|
+
with zipfile.ZipFile(self.inputs / 'project.zip', 'w') as archive:
|
|
57
|
+
for name, content in entries.items() if isinstance(entries, dict) else entries:
|
|
58
|
+
archive.writestr(name, content)
|
|
59
|
+
response = io.StringIO()
|
|
60
|
+
serve(io.BytesIO(json.dumps({'operation': 'preview.build', 'input': 'project.zip', 'output': output}).encode()),
|
|
61
|
+
response, load().commands, self.inputs, self.outputs)
|
|
62
|
+
return json.loads(response.getvalue())
|
|
63
|
+
|
|
64
|
+
def assert_failure(self, entries, code, output='preview.html'):
|
|
65
|
+
result = self.invoke(entries, output)
|
|
66
|
+
self.assertFalse(result['success'], result)
|
|
67
|
+
self.assertEqual(result['error']['code'], code, result)
|
|
68
|
+
self.assertFalse((self.outputs / output).exists())
|
|
69
|
+
|
|
70
|
+
def assert_counter_click(self, html, expected):
|
|
71
|
+
scripts = Scripts()
|
|
72
|
+
scripts.feed(html)
|
|
73
|
+
# VM 只验证产物脚本的一次交互;真实隔离浏览器验收另由 App E2E 执行。
|
|
74
|
+
program = '''
|
|
75
|
+
const vm = require('node:vm');
|
|
76
|
+
const assert = require('node:assert/strict');
|
|
77
|
+
const button = { textContent: '0' };
|
|
78
|
+
const context = vm.createContext({ document: { getElementById() { return button; } } });
|
|
79
|
+
for (const script of JSON.parse(process.argv[1])) vm.runInContext(script, context, { timeout: 1000 });
|
|
80
|
+
button.onclick({ currentTarget: button });
|
|
81
|
+
assert.equal(button.textContent, process.argv[2]);
|
|
82
|
+
'''
|
|
83
|
+
with (self.root / 'interaction.log').open('w+') as log:
|
|
84
|
+
result = subprocess.run(['node', '-e', program, json.dumps(scripts.scripts), str(expected)],
|
|
85
|
+
stdout=log, stderr=log, timeout=10)
|
|
86
|
+
log.seek(0)
|
|
87
|
+
self.assertEqual(result.returncode, 0, log.read())
|
|
88
|
+
|
|
89
|
+
def test_builds_vite_style_local_modules_and_css_without_running_project_config(self):
|
|
90
|
+
result = self.invoke({
|
|
91
|
+
'index.html': '<!doctype html><html><head><link rel="stylesheet" href="/src/style.css"></head><body><button id="count">0</button><script type="module" src="/src/main.ts"></script></body></html>',
|
|
92
|
+
'src/main.ts': 'import { next } from "./counter.ts"; document.getElementById("count")!.onclick = event => { event.currentTarget.textContent = String(next()); };',
|
|
93
|
+
'src/counter.ts': 'let count: number = 0; export function next(): number { return ++count; }',
|
|
94
|
+
'src/style.css': '@import "./colors.css"; button { color: var(--accent); }',
|
|
95
|
+
'src/colors.css': ':root { --accent: #123456; }',
|
|
96
|
+
'package.json': '{"scripts":{"build":"exit 99"},"dependencies":{"unrelated-package":"*"}}',
|
|
97
|
+
'vite.config.js': 'throw new Error("Project config must never run");',
|
|
98
|
+
'tsconfig.json': '{"compilerOptions":{"plugins":[{"name":"must-never-load"}]}}',
|
|
99
|
+
})
|
|
100
|
+
self.assertTrue(result['success'], result)
|
|
101
|
+
self.assertEqual(result['data']['artifact']['name'], 'preview.html')
|
|
102
|
+
output = self.outputs / 'preview.html'
|
|
103
|
+
self.assertEqual(result['data']['artifact']['bytes'], output.stat().st_size)
|
|
104
|
+
html = output.read_text()
|
|
105
|
+
self.assertIn('id="count"', html)
|
|
106
|
+
self.assertIn('#123456', html)
|
|
107
|
+
self.assertNotIn('src="/src/main.ts"', html)
|
|
108
|
+
self.assertNotIn('href="/src/style.css"', html)
|
|
109
|
+
self.assertNotIn('must-never-load', html)
|
|
110
|
+
self.assert_counter_click(html, 1)
|
|
111
|
+
|
|
112
|
+
def test_rebuild_uses_changed_module_without_previous_result(self):
|
|
113
|
+
entries = {
|
|
114
|
+
'index.html': '<button id="count">0</button><script type="module" src="/src/main.js"></script>',
|
|
115
|
+
'src/main.js': 'import { next } from "./counter.js"; document.getElementById("count").onclick = event => { event.currentTarget.textContent = String(next()); };',
|
|
116
|
+
'src/counter.js': 'export const next = () => 1;',
|
|
117
|
+
}
|
|
118
|
+
first = self.invoke(entries, 'first.html')
|
|
119
|
+
self.assertTrue(first['success'], first)
|
|
120
|
+
self.assert_counter_click((self.outputs / 'first.html').read_text(), 1)
|
|
121
|
+
entries['src/counter.js'] = 'export const next = () => 2;'
|
|
122
|
+
second = self.invoke(entries, 'second.html')
|
|
123
|
+
self.assertTrue(second['success'], second)
|
|
124
|
+
self.assert_counter_click((self.outputs / 'second.html').read_text(), 2)
|
|
125
|
+
|
|
126
|
+
def test_ignores_project_package_side_effect_and_browser_resolution_settings(self):
|
|
127
|
+
result = self.invoke({
|
|
128
|
+
'index.html': '<button id="count">0</button><script type="module" src="/main.js"></script>',
|
|
129
|
+
'main.js': 'import "./setup.js";',
|
|
130
|
+
'setup.js': 'document.getElementById("count").onclick = event => { event.currentTarget.textContent = "1"; };',
|
|
131
|
+
'package.json': '{"sideEffects":false,"browser":{"./setup.js":"./wrong.js"}}',
|
|
132
|
+
'wrong.js': 'throw new Error("Project package settings must never be used");',
|
|
133
|
+
})
|
|
134
|
+
self.assertTrue(result['success'], result)
|
|
135
|
+
self.assert_counter_click((self.outputs / 'preview.html').read_text(), 1)
|
|
136
|
+
|
|
137
|
+
def test_embeds_local_html_javascript_css_and_svelte_assets(self):
|
|
138
|
+
result = self.invoke({
|
|
139
|
+
'index.html': '<img src="/src/logo.svg"><div id="app"></div><link rel="stylesheet" href="/src/style.css"><script type="module" src="/src/main.ts"></script>',
|
|
140
|
+
'src/main.ts': 'import { mount } from "svelte"; import App from "./App.svelte"; import logo from "./logo.svg"; mount(App, { target: document.getElementById("app")!, props: { logo } });',
|
|
141
|
+
'src/App.svelte': '<script>import { title } from "./label.js"; let { logo } = $props(); let count = $state(0);</script><button onclick={() => count++}>{title}: {count}</button><img src={logo} alt="Logo"><style>button { background-image: url("./logo.svg"); }</style>',
|
|
142
|
+
'src/label.js': 'export const title = "Local Svelte counter";',
|
|
143
|
+
'src/style.css': 'body { background-image: url("./logo.svg"); }',
|
|
144
|
+
'src/logo.svg': '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1 1"><path d="M0 0h1v1H0z"/></svg>',
|
|
145
|
+
})
|
|
146
|
+
self.assertTrue(result['success'], result)
|
|
147
|
+
html = (self.outputs / 'preview.html').read_text()
|
|
148
|
+
self.assertIn('Local Svelte counter', html)
|
|
149
|
+
self.assertIn('data:image/svg+xml', html)
|
|
150
|
+
self.assertNotIn('url("./logo.svg")', html)
|
|
151
|
+
self.assertNotIn('from "svelte"', html)
|
|
152
|
+
|
|
153
|
+
def test_embeds_svelte_static_src_and_poster_using_component_relative_paths(self):
|
|
154
|
+
result = self.invoke({
|
|
155
|
+
'index.html': '<div id="app"></div><script type="module" src="/src/main.js"></script>',
|
|
156
|
+
'src/main.js': 'import { mount } from "svelte"; import App from "./App.svelte"; mount(App, { target: document.getElementById("app") });',
|
|
157
|
+
'src/App.svelte': '<script>let visible = $state(true);</script>{#if visible}<img src="/src/logo.svg" alt="Logo"><img src={"./nested/poster.svg"} alt="Poster"><video poster="./nested/poster.svg"></video>{/if}',
|
|
158
|
+
'src/logo.svg': '<svg xmlns="http://www.w3.org/2000/svg"><title>Root logo</title></svg>',
|
|
159
|
+
'src/nested/poster.svg': '<svg xmlns="http://www.w3.org/2000/svg"><title>Relative poster</title></svg>',
|
|
160
|
+
})
|
|
161
|
+
self.assertTrue(result['success'], result)
|
|
162
|
+
html = (self.outputs / 'preview.html').read_text()
|
|
163
|
+
self.assertTrue('/src/logo.svg' not in html, 'Static component src was not bundled')
|
|
164
|
+
self.assertTrue('./nested/poster.svg' not in html, 'Static component poster was not bundled')
|
|
165
|
+
self.assertIn('data:image/svg+xml;base64,', html)
|
|
166
|
+
|
|
167
|
+
def test_rejects_svelte_unsupported_srcset_and_nonlocal_static_assets(self):
|
|
168
|
+
for index, component in enumerate(['<img srcset="/src/logo.svg 1x, /src/logo.svg 2x" alt="Logo">',
|
|
169
|
+
'<img src="https://example.com/logo.svg" alt="Logo">',
|
|
170
|
+
'<video poster="../../outside.png"></video>']):
|
|
171
|
+
with self.subTest(component=component):
|
|
172
|
+
self.assert_failure({
|
|
173
|
+
'index.html': '<div id="app"></div><script type="module" src="/main.js"></script>',
|
|
174
|
+
'main.js': 'import { mount } from "svelte"; import App from "./src/App.svelte"; mount(App, { target: document.getElementById("app") });',
|
|
175
|
+
'src/App.svelte': component,
|
|
176
|
+
'src/logo.svg': '<svg xmlns="http://www.w3.org/2000/svg"></svg>',
|
|
177
|
+
}, 'UNSUPPORTED_PROJECT', f'component-{index}.html')
|
|
178
|
+
|
|
179
|
+
def test_rejects_network_dependencies_dynamic_imports_and_escapes(self):
|
|
180
|
+
imports = ['https://example.com/x.js', '//example.com/x.js', 'file:///etc/passwd',
|
|
181
|
+
'../../outside.js', '/etc/passwd', 'react', 'node:fs', './missing.js', './%2e%2e/%2e%2e/outside.js']
|
|
182
|
+
for name in imports:
|
|
183
|
+
with self.subTest(import_path=name):
|
|
184
|
+
self.assert_failure({'index.html': '<script type="module" src="/main.js"></script>',
|
|
185
|
+
'main.js': 'import ' + json.dumps(name) + ';'}, 'UNSUPPORTED_PROJECT')
|
|
186
|
+
self.assert_failure({'index.html': '<script type="module">const name = location.hash; import(name);</script>'}, 'UNSUPPORTED_PROJECT')
|
|
187
|
+
self.assert_failure({'index.html': '<link rel="stylesheet" href="https://example.com/style.css">'}, 'UNSUPPORTED_PROJECT')
|
|
188
|
+
self.assert_failure({'index.html': '<style>body{background:url(//example.com/pixel.png)}</style>'}, 'UNSUPPORTED_PROJECT')
|
|
189
|
+
self.assert_failure({'index.html': '<img src="/src/logo.svg">',
|
|
190
|
+
'src/logo.svg': '<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>'}, 'UNSUPPORTED_PROJECT')
|
|
191
|
+
|
|
192
|
+
def test_rejects_unsafe_zip_entries_and_packaged_dependencies(self):
|
|
193
|
+
link = zipfile.ZipInfo('linked.js')
|
|
194
|
+
link.create_system = 3
|
|
195
|
+
link.external_attr = (stat.S_IFLNK | 0o777) << 16
|
|
196
|
+
cases = [([('../outside.js', 'export default 1')], 'UNSAFE_PROJECT'),
|
|
197
|
+
([('.env', 'PRIVATE=value')], 'UNSAFE_PROJECT'),
|
|
198
|
+
([('src/App.js', '1'), ('src/app.js', '2')], 'UNSAFE_PROJECT'),
|
|
199
|
+
([(link, 'outside.js')], 'UNSAFE_PROJECT'),
|
|
200
|
+
([('node_modules/anything/index.js', '1')], 'UNSUPPORTED_PROJECT')]
|
|
201
|
+
for extra, code in cases:
|
|
202
|
+
with self.subTest(entries=extra):
|
|
203
|
+
self.assert_failure([('index.html', '<h1>Hello</h1>')] + extra, code)
|
|
204
|
+
|
|
205
|
+
def test_applies_project_source_and_output_budgets(self):
|
|
206
|
+
self.assert_failure({'src/main.js': 'export default 1;'}, 'INVALID_PROJECT')
|
|
207
|
+
self.assert_failure({'index.html': '<h1>Hello</h1>', 'large.js': 'x' * (1024 * 1024 + 1)}, 'UNSAFE_PROJECT')
|
|
208
|
+
self.assert_failure({'index.html': '<h1>Hello</h1>', 'large.png': b'x' * (8 * 1024 * 1024)}, 'UNSAFE_PROJECT')
|
|
209
|
+
self.assert_failure({'index.html': '<img src="/large.png">', 'large.png': b'x' * (800 * 1024)}, 'OUTPUT_LIMIT')
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
if __name__ == '__main__':
|
|
213
|
+
unittest.main()
|
package/jobs/tool.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
"""固定工具镜像的单请求入口。"""
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import pathlib
|
|
6
|
+
import re
|
|
7
|
+
import shutil
|
|
8
|
+
import signal
|
|
9
|
+
import stat
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import tempfile
|
|
13
|
+
|
|
14
|
+
REQUEST_LIMIT = 64 * 1024
|
|
15
|
+
FILE_LIMIT = 50 * 1024 * 1024
|
|
16
|
+
TEXT_LIMIT = 256 * 1024
|
|
17
|
+
NAME = re.compile(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z', re.ASCII)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ToolError(Exception):
|
|
21
|
+
def __init__(self, code, message):
|
|
22
|
+
super().__init__(message)
|
|
23
|
+
self.code = code
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def require_name(name, code):
|
|
27
|
+
if not isinstance(name, str) or not NAME.fullmatch(name):
|
|
28
|
+
raise ToolError(code, 'A simple file name is required')
|
|
29
|
+
return name
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def input_file(root, name):
|
|
33
|
+
path = root / require_name(name, 'INVALID_INPUT')
|
|
34
|
+
try:
|
|
35
|
+
info = path.lstat()
|
|
36
|
+
except OSError:
|
|
37
|
+
raise ToolError('INVALID_INPUT', 'Input file is unavailable') from None
|
|
38
|
+
if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= FILE_LIMIT:
|
|
39
|
+
raise ToolError('INVALID_INPUT', 'Input is not a regular file or exceeds the size limit')
|
|
40
|
+
return path
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def output_file(root, name):
|
|
44
|
+
path = root / require_name(name, 'INVALID_OUTPUT')
|
|
45
|
+
if os.path.lexists(path):
|
|
46
|
+
raise ToolError('INVALID_OUTPUT', 'Output already exists')
|
|
47
|
+
return path
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def text_result(text):
|
|
51
|
+
if len(text.encode('utf-8')) > TEXT_LIMIT:
|
|
52
|
+
raise ToolError('OUTPUT_LIMIT', 'Extracted text exceeds the size limit')
|
|
53
|
+
return {'text': text}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def read_text(path):
|
|
57
|
+
with path.open('rb') as source:
|
|
58
|
+
data = source.read(TEXT_LIMIT + 1)
|
|
59
|
+
if len(data) > TEXT_LIMIT:
|
|
60
|
+
raise ToolError('OUTPUT_LIMIT', 'Extracted text exceeds the size limit')
|
|
61
|
+
return text_result(data.decode('utf-8'))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def publish(source, output, signature, signature_offset=0):
|
|
65
|
+
info = source.lstat()
|
|
66
|
+
if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= FILE_LIMIT:
|
|
67
|
+
raise ToolError('INVALID_OUTPUT', 'Produced artifact is not a bounded regular file')
|
|
68
|
+
with source.open('rb') as stream:
|
|
69
|
+
if not stream.read(signature_offset + len(signature))[signature_offset:].startswith(signature):
|
|
70
|
+
raise ToolError('INVALID_OUTPUT', 'Produced artifact has an unexpected format')
|
|
71
|
+
stream.seek(0)
|
|
72
|
+
try:
|
|
73
|
+
with output.open('xb') as destination:
|
|
74
|
+
try:
|
|
75
|
+
shutil.copyfileobj(stream, destination)
|
|
76
|
+
except OSError:
|
|
77
|
+
output.unlink(missing_ok=True)
|
|
78
|
+
raise
|
|
79
|
+
except FileExistsError:
|
|
80
|
+
raise ToolError('INVALID_OUTPUT', 'Output already exists') from None
|
|
81
|
+
return {'artifact': {'name': output.name, 'bytes': info.st_size}}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def run_command(args, timeout, workdir):
|
|
85
|
+
# 不继承代理、工具凭据或外部程序配置。诊断输出不能混入协议。
|
|
86
|
+
env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'HOME': str(workdir), 'TMPDIR': str(workdir), 'LANG': 'C.UTF-8'}
|
|
87
|
+
with subprocess.Popen(args, cwd=workdir, env=env, stdin=subprocess.DEVNULL,
|
|
88
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) as child:
|
|
89
|
+
try:
|
|
90
|
+
code = child.wait(timeout=timeout)
|
|
91
|
+
except BaseException:
|
|
92
|
+
try:
|
|
93
|
+
os.killpg(child.pid, signal.SIGKILL)
|
|
94
|
+
except ProcessLookupError:
|
|
95
|
+
pass
|
|
96
|
+
child.wait()
|
|
97
|
+
raise
|
|
98
|
+
if code != 0:
|
|
99
|
+
raise ToolError('TOOL_FAILED', 'The selected operation failed')
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def no_duplicates(pairs):
|
|
103
|
+
values = {}
|
|
104
|
+
for key, value in pairs:
|
|
105
|
+
if key in values:
|
|
106
|
+
raise ToolError('INVALID_REQUEST', 'Duplicate request field')
|
|
107
|
+
values[key] = value
|
|
108
|
+
return values
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def serve(source, response, commands, inputs, outputs):
|
|
112
|
+
try:
|
|
113
|
+
raw = source.read(REQUEST_LIMIT + 1)
|
|
114
|
+
if len(raw) > REQUEST_LIMIT:
|
|
115
|
+
raise ToolError('INVALID_REQUEST', 'Request exceeds the size limit')
|
|
116
|
+
request = json.loads(raw.decode('utf-8'), object_pairs_hook=no_duplicates)
|
|
117
|
+
operation = request.get('operation') if isinstance(request, dict) else None
|
|
118
|
+
if not isinstance(operation, str) or operation not in commands:
|
|
119
|
+
raise ToolError('INVALID_REQUEST', 'Unknown operation')
|
|
120
|
+
required, optional, handler = commands[operation]
|
|
121
|
+
keys = set(request) - {'operation'}
|
|
122
|
+
if not required <= keys <= required | optional:
|
|
123
|
+
raise ToolError('INVALID_REQUEST', 'Missing or unsupported request fields')
|
|
124
|
+
item = input_file(inputs, request['input']) if 'input' in request else None
|
|
125
|
+
artifact = output_file(outputs, request['output']) if 'output' in request else None
|
|
126
|
+
with tempfile.TemporaryDirectory(prefix='hai-tool-') as workspace:
|
|
127
|
+
data = handler(request, item, artifact, pathlib.Path(workspace))
|
|
128
|
+
result = {'success': True, 'data': data}
|
|
129
|
+
except ToolError as error:
|
|
130
|
+
result = {'success': False, 'error': {'code': error.code, 'message': str(error)}}
|
|
131
|
+
except (json.JSONDecodeError, UnicodeDecodeError, RecursionError):
|
|
132
|
+
result = {'success': False, 'error': {'code': 'INVALID_REQUEST', 'message': 'Malformed JSON request'}}
|
|
133
|
+
except subprocess.TimeoutExpired:
|
|
134
|
+
result = {'success': False, 'error': {'code': 'TIMEOUT', 'message': 'The selected operation timed out'}}
|
|
135
|
+
except (OSError, ValueError):
|
|
136
|
+
result = {'success': False, 'error': {'code': 'TOOL_FAILED', 'message': 'The selected operation failed'}}
|
|
137
|
+
response.write(json.dumps(result, ensure_ascii=True, allow_nan=False) + '\n')
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
if __name__ == '__main__':
|
|
141
|
+
# 可执行文件只加载构建时随镜像复制的固定操作。
|
|
142
|
+
sys.path.insert(0, '/opt/hai-tools')
|
|
143
|
+
sys.modules['tool'] = sys.modules[__name__]
|
|
144
|
+
from operation import commands
|
|
145
|
+
serve(sys.stdin.buffer, sys.stdout, commands, pathlib.Path('/input'), pathlib.Path('/output'))
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
FROM docker.io/library/node:24.22.1-bookworm-slim@sha256:be8204d4bd327363357f532187b752ff62759c31bedf98e9e18fc08c9cbd605c
|
|
2
|
+
RUN mkdir -p /etc/ssl/certs && node -e "require('node:fs').writeFileSync('/etc/ssl/certs/ca-certificates.crt', require('node:tls').rootCertificates.join('\\n'))"
|
|
3
|
+
ARG DEBIAN_MIRROR=https://deb.debian.org/debian
|
|
4
|
+
ARG DEBIAN_SECURITY_MIRROR=https://deb.debian.org/debian-security
|
|
5
|
+
RUN set -eu; \
|
|
6
|
+
sed -i -e "s|http://deb.debian.org/debian-security|${DEBIAN_SECURITY_MIRROR}|g" -e "s|http://deb.debian.org/debian|${DEBIAN_MIRROR}|g" /etc/apt/sources.list.d/debian.sources; \
|
|
7
|
+
apt-get -o Acquire::Retries=2 -o Acquire::https::Timeout=20 update; \
|
|
8
|
+
apt-get install -y --no-install-recommends python3 chromium ffmpeg fonts-liberation fonts-noto-cjk; \
|
|
9
|
+
rm -rf /var/lib/apt/lists/*
|
|
10
|
+
RUN mkdir -p /opt/hai-tools /input /output && chown 1000:1000 /output
|
|
11
|
+
COPY tool.py /usr/local/bin/hai-tool
|
|
12
|
+
COPY media_files.py hyperframes-runtime.mjs /opt/hai-tools/
|
|
13
|
+
COPY video_render /opt/hai-tools/video_render
|
|
14
|
+
RUN printf 'from video_render.operation import commands\n' > /opt/hai-tools/operation.py
|
|
15
|
+
RUN npm install --prefix /opt/hai-tools --save-exact --no-audit --no-fund hyperframes@0.9.56
|
|
16
|
+
RUN chmod 0555 /usr/local/bin/hai-tool
|
|
17
|
+
USER 1000:1000
|
|
18
|
+
ENV HOME=/tmp PYTHONDONTWRITEBYTECODE=1
|
|
19
|
+
WORKDIR /tmp
|
|
20
|
+
ENTRYPOINT ["/usr/local/bin/hai-tool"]
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""离线视频渲染作业。"""
|
|
2
|
+
import math
|
|
3
|
+
import pathlib
|
|
4
|
+
import re
|
|
5
|
+
from html.parser import HTMLParser
|
|
6
|
+
from urllib.parse import unquote, urlsplit
|
|
7
|
+
from tool import ToolError, publish, run_command
|
|
8
|
+
from media_files import extract_project
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def local_resource(value, directory, base):
|
|
12
|
+
if not value or value.startswith('#'):
|
|
13
|
+
return
|
|
14
|
+
if '\\' in value or any(ord(character) < 32 for character in value):
|
|
15
|
+
raise ToolError('INVALID_PROJECT', 'Project resource has an unsafe path')
|
|
16
|
+
parsed = urlsplit(value)
|
|
17
|
+
path = pathlib.PurePosixPath(unquote(parsed.path))
|
|
18
|
+
if parsed.scheme or parsed.netloc or path.is_absolute():
|
|
19
|
+
raise ToolError('INVALID_PROJECT', 'Project resources must use local relative files')
|
|
20
|
+
target = base.joinpath(*path.parts).resolve()
|
|
21
|
+
if not target.is_relative_to(directory.resolve()) or not target.is_file():
|
|
22
|
+
raise ToolError('INVALID_PROJECT', 'Project resource is missing or escapes the project')
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def validate_css(source, directory, base):
|
|
26
|
+
for value in re.findall(r'''url\(\s*["']?([^"')]+)''', source, flags=re.IGNORECASE):
|
|
27
|
+
local_resource(value.strip(), directory, base)
|
|
28
|
+
for value in re.findall(r'''@import\s+["']([^"']+)["']''', source, flags=re.IGNORECASE):
|
|
29
|
+
local_resource(value, directory, base)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Composition(HTMLParser):
|
|
33
|
+
def __init__(self, directory, base):
|
|
34
|
+
super().__init__()
|
|
35
|
+
self.attributes = None
|
|
36
|
+
self.directory = directory
|
|
37
|
+
self.base = base
|
|
38
|
+
self.in_style = False
|
|
39
|
+
|
|
40
|
+
def handle_starttag(self, tag, attributes):
|
|
41
|
+
values = dict(attributes)
|
|
42
|
+
# 浏览器采用首个重复属性,dict 则保留最后一个;拒绝歧义以保持校验与渲染一致。
|
|
43
|
+
if len(values) != len(attributes):
|
|
44
|
+
raise ToolError('INVALID_PROJECT', 'Project HTML contains duplicate attributes')
|
|
45
|
+
if self.attributes is None and 'data-composition-id' in values:
|
|
46
|
+
self.attributes = values
|
|
47
|
+
for key in ('src', 'poster', 'data-composition-src'):
|
|
48
|
+
if values.get(key):
|
|
49
|
+
local_resource(values[key], self.directory, self.base)
|
|
50
|
+
if tag in ('link', 'base') and values.get('href'):
|
|
51
|
+
local_resource(values['href'], self.directory, self.base)
|
|
52
|
+
if values.get('style'):
|
|
53
|
+
validate_css(values['style'], self.directory, self.base)
|
|
54
|
+
for item in values.get('srcset', '').split(','):
|
|
55
|
+
if item.strip():
|
|
56
|
+
local_resource(item.split()[0], self.directory, self.base)
|
|
57
|
+
if tag == 'style':
|
|
58
|
+
self.in_style = True
|
|
59
|
+
|
|
60
|
+
def handle_endtag(self, tag):
|
|
61
|
+
if tag == 'style':
|
|
62
|
+
self.in_style = False
|
|
63
|
+
|
|
64
|
+
def handle_data(self, source):
|
|
65
|
+
if self.in_style:
|
|
66
|
+
validate_css(source, self.directory, self.base)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def validate_project(directory):
|
|
70
|
+
source = directory / 'index.html'
|
|
71
|
+
if not source.is_file() or source.stat().st_size > 2 * 1024 * 1024:
|
|
72
|
+
raise ToolError('INVALID_PROJECT', 'Project must contain a bounded index.html')
|
|
73
|
+
document = Composition(directory, source.parent)
|
|
74
|
+
document.feed(source.read_text(encoding='utf-8'))
|
|
75
|
+
try:
|
|
76
|
+
values = document.attributes or {}
|
|
77
|
+
duration = float(values['data-duration'])
|
|
78
|
+
width = int(values['data-width'])
|
|
79
|
+
height = int(values['data-height'])
|
|
80
|
+
start = float(values.get('data-start', '0'))
|
|
81
|
+
if (not math.isfinite(duration) or not 0 < duration <= 30 or start != 0
|
|
82
|
+
or not 16 <= width <= 1920 or not 16 <= height <= 1920
|
|
83
|
+
or width * height > 1920 * 1080 or width % 2 or height % 2):
|
|
84
|
+
raise ValueError
|
|
85
|
+
except (KeyError, TypeError, ValueError):
|
|
86
|
+
raise ToolError('INVALID_PROJECT', 'Composition duration or dimensions exceed the supported limits') from None
|
|
87
|
+
for file in directory.rglob('*'):
|
|
88
|
+
if file.suffix.lower() == '.css':
|
|
89
|
+
validate_css(file.read_text(encoding='utf-8'), directory, file.parent)
|
|
90
|
+
elif file.suffix.lower() == '.html' and file != source:
|
|
91
|
+
Composition(directory, file.parent).feed(file.read_text(encoding='utf-8'))
|
|
92
|
+
return duration, width, height
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def render(request, source, artifact, workspace):
|
|
96
|
+
fps = request.get('fps', 30)
|
|
97
|
+
if type(fps) is not int or fps not in (24, 25, 30) or source.suffix.lower() != '.zip' or artifact.suffix.lower() != '.mp4':
|
|
98
|
+
raise ToolError('INVALID_REQUEST', 'Video rendering requires a ZIP project, MP4 output and 24, 25 or 30 fps')
|
|
99
|
+
directory = workspace / 'project'
|
|
100
|
+
directory.mkdir()
|
|
101
|
+
extract_project(source, directory)
|
|
102
|
+
duration, width, height = validate_project(directory)
|
|
103
|
+
worker = pathlib.Path(__file__).with_name('render.mjs')
|
|
104
|
+
result = workspace / 'result.mp4'
|
|
105
|
+
run_command(['node', str(worker), str(directory), str(result), str(fps), str(duration), str(width), str(height)], 300, workspace)
|
|
106
|
+
return publish(result, artifact, b'ftyp', 4)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
commands = {'video.render': ({'input', 'output'}, {'fps'}, render)}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { dirname } from 'node:path'
|
|
2
|
+
import process from 'node:process'
|
|
3
|
+
import { probeVideo, runHyperframes } from '../hyperframes-runtime.mjs'
|
|
4
|
+
|
|
5
|
+
async function render() {
|
|
6
|
+
const [project, output, fps, duration, width, height] = process.argv.slice(2)
|
|
7
|
+
await runHyperframes([
|
|
8
|
+
'render',
|
|
9
|
+
project,
|
|
10
|
+
'--output',
|
|
11
|
+
output,
|
|
12
|
+
'--fps',
|
|
13
|
+
fps,
|
|
14
|
+
'--format',
|
|
15
|
+
'mp4',
|
|
16
|
+
'--quality',
|
|
17
|
+
'standard',
|
|
18
|
+
'--workers',
|
|
19
|
+
'1',
|
|
20
|
+
'--video-bitrate',
|
|
21
|
+
'8M',
|
|
22
|
+
'--no-browser-gpu',
|
|
23
|
+
'--strict',
|
|
24
|
+
], dirname(project))
|
|
25
|
+
await probeVideo(output, dirname(project), {
|
|
26
|
+
fps: Number(fps),
|
|
27
|
+
duration: Number(duration),
|
|
28
|
+
width: Number(width),
|
|
29
|
+
height: Number(height),
|
|
30
|
+
})
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
render().catch(() => {
|
|
34
|
+
process.stderr.write('HyperFrames video.render failed\n')
|
|
35
|
+
process.exitCode = 1
|
|
36
|
+
})
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
FROM docker.io/library/node:24.22.1-bookworm-slim@sha256:be8204d4bd327363357f532187b752ff62759c31bedf98e9e18fc08c9cbd605c
|
|
2
|
+
RUN mkdir -p /etc/ssl/certs && node -e "require('node:fs').writeFileSync('/etc/ssl/certs/ca-certificates.crt', require('node:tls').rootCertificates.join('\\n'))"
|
|
3
|
+
ARG DEBIAN_MIRROR=https://deb.debian.org/debian
|
|
4
|
+
ARG DEBIAN_SECURITY_MIRROR=https://deb.debian.org/debian-security
|
|
5
|
+
RUN set -eu; \
|
|
6
|
+
sed -i -e "s|http://deb.debian.org/debian-security|${DEBIAN_SECURITY_MIRROR}|g" -e "s|http://deb.debian.org/debian|${DEBIAN_MIRROR}|g" /etc/apt/sources.list.d/debian.sources; \
|
|
7
|
+
apt-get -o Acquire::Retries=2 -o Acquire::https::Timeout=20 update; \
|
|
8
|
+
apt-get install -y --no-install-recommends python3 chromium fonts-liberation fonts-noto-cjk; \
|
|
9
|
+
rm -rf /var/lib/apt/lists/*
|
|
10
|
+
RUN mkdir -p /opt/hai-tools /input /output && chown 1000:1000 /output
|
|
11
|
+
COPY tool.py /usr/local/bin/hai-tool
|
|
12
|
+
COPY browser_policy.py media_files.py hyperframes-runtime.mjs /opt/hai-tools/
|
|
13
|
+
COPY web_capture /opt/hai-tools/web_capture
|
|
14
|
+
RUN printf 'from web_capture.operation import commands\n' > /opt/hai-tools/operation.py
|
|
15
|
+
RUN npm install --prefix /opt/hai-tools --save-exact --no-audit --no-fund hyperframes@0.9.56
|
|
16
|
+
RUN chmod 0555 /usr/local/bin/hai-tool
|
|
17
|
+
USER 1000:1000
|
|
18
|
+
ENV HOME=/tmp PYTHONDONTWRITEBYTECODE=1
|
|
19
|
+
WORKDIR /tmp
|
|
20
|
+
ENTRYPOINT ["/usr/local/bin/hai-tool"]
|