openclacky 1.3.11 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/lib/clacky/agent/skill_manager.rb +1 -0
- data/lib/clacky/default_extensions/ext-studio/api/handler.rb +58 -2
- data/lib/clacky/default_extensions/ext-studio/panels/studio/view.js +355 -44
- data/lib/clacky/default_skills/media-gen/SKILL.md +182 -7
- data/lib/clacky/default_skills/media-gen/scripts/video_seq.sh +5 -3
- data/lib/clacky/extension/packager.rb +11 -0
- data/lib/clacky/extension/verifier.rb +1 -1
- data/lib/clacky/media/generator.rb +41 -0
- data/lib/clacky/media/volcengine.rb +394 -0
- data/lib/clacky/server/http_server.rb +48 -4
- data/lib/clacky/tools/terminal.rb +24 -8
- data/lib/clacky/ui2/layout_manager.rb +4 -0
- data/lib/clacky/ui2/screen_buffer.rb +31 -3
- data/lib/clacky/utils/file_processor.rb +13 -8
- data/lib/clacky/version.rb +1 -1
- data/lib/clacky/web/app.css +110 -16
- data/lib/clacky/web/features/extensions/view.js +42 -7
- data/lib/clacky/web/features/skills/view.js +61 -1
- data/lib/clacky/web/i18n.js +24 -0
- data/lib/clacky/web/index.html +6 -11
- data/lib/clacky/web/sessions.js +12 -3
- metadata +2 -1
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "faraday"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
require "base64"
|
|
7
|
+
require_relative "base"
|
|
8
|
+
|
|
9
|
+
module Clacky
|
|
10
|
+
module Media
|
|
11
|
+
# Volcengine Ark (ByteDance Doubao Seedance) video generation provider.
|
|
12
|
+
#
|
|
13
|
+
# Ark is NOT OpenAI-compatible for video. It uses an asynchronous task
|
|
14
|
+
# model with its own request envelope: submit a task to
|
|
15
|
+
# POST /api/v3/contents/generations/tasks, then poll
|
|
16
|
+
# GET /api/v3/contents/generations/tasks/{id} until it succeeds, fails,
|
|
17
|
+
# or expires. Unlike the blocking Veo path, we surface that async model
|
|
18
|
+
# end to end: #generate_video submits and returns the task id immediately,
|
|
19
|
+
# and #video_status polls a single time (downloading the MP4 once the task
|
|
20
|
+
# has succeeded). This keeps a slow (e.g. 4k) render from blocking the HTTP
|
|
21
|
+
# request past its timeout and dropping the task id.
|
|
22
|
+
#
|
|
23
|
+
# Routing: Generator sends any base_url under *.volces.com here. We
|
|
24
|
+
# derive the API root from the host so users can paste the standard
|
|
25
|
+
# base_url they use for Ark chat models (…/api/v3) and still get working
|
|
26
|
+
# video generation.
|
|
27
|
+
#
|
|
28
|
+
# Seedance content is multimodal: alongside the text prompt, callers may
|
|
29
|
+
# attach a first frame, a last frame, reference images (0-9), reference
|
|
30
|
+
# videos (0-3) and reference audios (0-3). Each media item may be a
|
|
31
|
+
# public http(s) URL, a data URL, a local file path (encoded to a data
|
|
32
|
+
# URL), or a { "b64_json" => ..., "mime_type" => ... } hash.
|
|
33
|
+
class Volcengine < Base
|
|
34
|
+
TASKS_PATH = "/api/v3/contents/generations/tasks"
|
|
35
|
+
PROVIDER_ID = "volcengine"
|
|
36
|
+
|
|
37
|
+
# aspect_ratio -> Ark ratio. Ark also accepts more granular ratios and
|
|
38
|
+
# "adaptive"; when the caller passes one of those verbatim we forward it
|
|
39
|
+
# unchanged (see #resolve_ratio).
|
|
40
|
+
ASPECT_TO_RATIO = {
|
|
41
|
+
"landscape" => "16:9",
|
|
42
|
+
"portrait" => "9:16",
|
|
43
|
+
"square" => "1:1"
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
46
|
+
# Ratios Ark accepts directly. Anything else falls back to the
|
|
47
|
+
# aspect_ratio mapping / default.
|
|
48
|
+
ARK_RATIOS = %w[16:9 4:3 1:1 3:4 9:16 21:9 adaptive].freeze
|
|
49
|
+
|
|
50
|
+
DEFAULT_RATIO = "16:9"
|
|
51
|
+
|
|
52
|
+
# Explicit cost-control default: when the caller omits resolution we pin
|
|
53
|
+
# 720p rather than relying on Ark's server-side default (which could
|
|
54
|
+
# change and silently raise the user's bill).
|
|
55
|
+
DEFAULT_RESOLUTION = "720p"
|
|
56
|
+
|
|
57
|
+
# @param prompt [String] the video prompt (may be empty when driven
|
|
58
|
+
# purely by reference media, but Ark still recommends text)
|
|
59
|
+
# @param aspect_ratio [String] "landscape"/"portrait"/"square", or an
|
|
60
|
+
# Ark ratio string ("16:9", "9:16", "adaptive", ...)
|
|
61
|
+
# @param duration_seconds [Integer, nil] target duration; -1 lets the
|
|
62
|
+
# model choose (Seedance 2.0 / 1.5 Pro only)
|
|
63
|
+
# @param first_frame [String, Hash, nil] first frame image
|
|
64
|
+
# @param last_frame [String, Hash, nil] last frame image
|
|
65
|
+
# @param reference_images [Array, String, Hash, nil] reference images
|
|
66
|
+
# @param reference_videos [Array, String, Hash, nil] reference videos
|
|
67
|
+
# @param reference_audios [Array, String, Hash, nil] reference audios
|
|
68
|
+
# @param resolution [String, nil] "480p"/"720p"/"1080p"/"4k"
|
|
69
|
+
# @param generate_audio [Boolean, nil] whether to synthesize audio
|
|
70
|
+
# @param watermark [Boolean, nil] whether to add a watermark
|
|
71
|
+
# @param seed [Integer, nil] random seed
|
|
72
|
+
def generate_video(prompt:, aspect_ratio: "landscape", duration_seconds: nil, output_dir: nil,
|
|
73
|
+
image: nil, first_frame: nil, last_frame: nil,
|
|
74
|
+
reference_images: nil, reference_videos: nil, reference_audios: nil,
|
|
75
|
+
resolution: nil, generate_audio: nil, watermark: nil, seed: nil, **_kwargs)
|
|
76
|
+
# `image` is the legacy single first-frame field used across the media
|
|
77
|
+
# stack; treat it as first_frame when no explicit first_frame is given.
|
|
78
|
+
first_frame ||= image
|
|
79
|
+
|
|
80
|
+
content, build_err = build_content(
|
|
81
|
+
prompt: prompt,
|
|
82
|
+
first_frame: first_frame,
|
|
83
|
+
last_frame: last_frame,
|
|
84
|
+
reference_images: reference_images,
|
|
85
|
+
reference_videos: reference_videos,
|
|
86
|
+
reference_audios: reference_audios
|
|
87
|
+
)
|
|
88
|
+
if build_err
|
|
89
|
+
return video_error_response(
|
|
90
|
+
error: build_err, error_type: "invalid_argument",
|
|
91
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
92
|
+
)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
has_text = prompt.to_s.strip != ""
|
|
96
|
+
has_media = content.length > (has_text ? 1 : 0)
|
|
97
|
+
if !has_text && !has_media
|
|
98
|
+
return video_error_response(
|
|
99
|
+
error: "A text prompt or at least one reference image/video is required",
|
|
100
|
+
error_type: "invalid_argument",
|
|
101
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
102
|
+
)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
if @api_key.to_s.empty?
|
|
106
|
+
return video_error_response(
|
|
107
|
+
error: "api_key not configured for video model '#{@model}'",
|
|
108
|
+
error_type: "auth_required",
|
|
109
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
110
|
+
)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
ratio = resolve_ratio(aspect_ratio)
|
|
114
|
+
payload = { model: @model, content: content }
|
|
115
|
+
payload[:ratio] = ratio unless ratio.nil?
|
|
116
|
+
payload[:duration] = duration_seconds.to_i if duration_seconds
|
|
117
|
+
payload[:resolution] = (resolution && !resolution.to_s.empty?) ? resolution.to_s : DEFAULT_RESOLUTION
|
|
118
|
+
payload[:generate_audio] = generate_audio unless generate_audio.nil?
|
|
119
|
+
payload[:watermark] = watermark unless watermark.nil?
|
|
120
|
+
payload[:seed] = seed.to_i if seed
|
|
121
|
+
|
|
122
|
+
begin
|
|
123
|
+
response = connection.post(TASKS_PATH) do |req|
|
|
124
|
+
req.headers["Content-Type"] = "application/json"
|
|
125
|
+
req.headers["Authorization"] = "Bearer #{@api_key}"
|
|
126
|
+
req.body = JSON.generate(payload)
|
|
127
|
+
end
|
|
128
|
+
rescue Faraday::Error => e
|
|
129
|
+
return video_error_response(
|
|
130
|
+
error: "HTTP request failed: #{e.message}",
|
|
131
|
+
error_type: "network_error",
|
|
132
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
133
|
+
)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
body = parse_json(response.body)
|
|
137
|
+
unless body.is_a?(Hash)
|
|
138
|
+
return video_error_response(
|
|
139
|
+
error: "Invalid JSON response from upstream",
|
|
140
|
+
error_type: "invalid_response",
|
|
141
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
142
|
+
)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Ark reports submission errors via a nested "error" object.
|
|
146
|
+
if body["error"].is_a?(Hash)
|
|
147
|
+
err = body["error"]
|
|
148
|
+
return video_error_response(
|
|
149
|
+
error: "Upstream error #{err["code"]}: #{err["message"]}",
|
|
150
|
+
error_type: "api_error",
|
|
151
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
152
|
+
)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
unless response.success?
|
|
156
|
+
return video_error_response(
|
|
157
|
+
error: "Upstream #{response.status}: #{truncate(response.body, 500)}",
|
|
158
|
+
error_type: "api_error",
|
|
159
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
160
|
+
)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
task_id = body["id"]
|
|
164
|
+
if task_id.nil? || task_id.to_s.empty?
|
|
165
|
+
return video_error_response(
|
|
166
|
+
error: "Upstream did not return a task id",
|
|
167
|
+
error_type: "empty_response",
|
|
168
|
+
provider: PROVIDER_ID, prompt: prompt, aspect_ratio: aspect_ratio
|
|
169
|
+
)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Submit only: return the task id immediately. The caller polls
|
|
173
|
+
# GET /api/media/video/status so a long (e.g. 4k) render never blocks
|
|
174
|
+
# the HTTP request past its timeout — which used to drop the task id
|
|
175
|
+
# and push the model into resubmitting, doubling the bill.
|
|
176
|
+
{
|
|
177
|
+
"success" => true,
|
|
178
|
+
"status" => "submitted",
|
|
179
|
+
"task_id" => task_id,
|
|
180
|
+
"provider" => PROVIDER_ID,
|
|
181
|
+
"model" => @model,
|
|
182
|
+
"prompt" => prompt,
|
|
183
|
+
"aspect_ratio" => aspect_ratio,
|
|
184
|
+
"ratio" => ratio,
|
|
185
|
+
"duration_seconds" => (duration_seconds ? duration_seconds.to_i : nil)
|
|
186
|
+
}.compact
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Poll a previously submitted task once. On "succeeded" the MP4 is
|
|
190
|
+
# downloaded and a local path is returned; other states map to
|
|
191
|
+
# running / failed without blocking.
|
|
192
|
+
def video_status(task_id:, output_dir: nil)
|
|
193
|
+
if task_id.to_s.strip.empty?
|
|
194
|
+
return video_error_response(
|
|
195
|
+
error: "task_id is required", error_type: "invalid_argument",
|
|
196
|
+
provider: PROVIDER_ID
|
|
197
|
+
)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
state, detail = fetch_task(task_id)
|
|
201
|
+
case state
|
|
202
|
+
when :running
|
|
203
|
+
{ "success" => true, "status" => "running", "task_id" => task_id, "provider" => PROVIDER_ID, "model" => @model }
|
|
204
|
+
when :succeeded
|
|
205
|
+
local_path = save_image_from_url(detail, output_dir: output_dir || Dir.pwd, prefix: "vid", extension: "mp4")
|
|
206
|
+
if local_path.nil?
|
|
207
|
+
return {
|
|
208
|
+
"success" => false, "status" => "failed", "task_id" => task_id, "provider" => PROVIDER_ID,
|
|
209
|
+
"model" => @model, "error" => "Failed to download generated video from #{detail}",
|
|
210
|
+
"error_type" => "download_failed"
|
|
211
|
+
}
|
|
212
|
+
end
|
|
213
|
+
{
|
|
214
|
+
"success" => true, "status" => "succeeded", "video" => local_path,
|
|
215
|
+
"task_id" => task_id, "provider" => PROVIDER_ID, "model" => @model
|
|
216
|
+
}
|
|
217
|
+
else # :failed
|
|
218
|
+
{
|
|
219
|
+
"success" => false, "status" => "failed", "task_id" => task_id, "provider" => PROVIDER_ID,
|
|
220
|
+
"model" => @model, "error" => detail, "error_type" => "task_failed"
|
|
221
|
+
}
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# Query a task once (no sleeping/looping). Returns one of:
|
|
226
|
+
# [:running, nil]
|
|
227
|
+
# [:succeeded, video_url]
|
|
228
|
+
# [:failed, error_message]
|
|
229
|
+
private def fetch_task(task_id)
|
|
230
|
+
begin
|
|
231
|
+
resp = connection.get("#{TASKS_PATH}/#{task_id}") do |req|
|
|
232
|
+
req.headers["Authorization"] = "Bearer #{@api_key}"
|
|
233
|
+
end
|
|
234
|
+
rescue Faraday::Error => e
|
|
235
|
+
return [:failed, "Polling request failed: #{e.message}"]
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
task = parse_json(resp.body)
|
|
239
|
+
return [:failed, "Invalid polling response JSON"] unless task.is_a?(Hash)
|
|
240
|
+
|
|
241
|
+
case task["status"]
|
|
242
|
+
when "succeeded"
|
|
243
|
+
url = task.dig("content", "video_url")
|
|
244
|
+
return [:failed, "Task succeeded but returned no video_url"] if url.nil? || url.empty?
|
|
245
|
+
[:succeeded, url]
|
|
246
|
+
when "failed"
|
|
247
|
+
msg = task.dig("error", "message") || task["error"] || "Unknown error"
|
|
248
|
+
[:failed, "Task failed: #{msg}"]
|
|
249
|
+
when "cancelled", "canceled"
|
|
250
|
+
[:failed, "Task was cancelled"]
|
|
251
|
+
when "expired"
|
|
252
|
+
[:failed, "Task expired before completion"]
|
|
253
|
+
else # "queued" / "running"
|
|
254
|
+
[:running, nil]
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Build the Ark content[] array. Returns [content, nil] on success or
|
|
259
|
+
# [nil, error_message] when a media item can't be resolved.
|
|
260
|
+
private def build_content(prompt:, first_frame:, last_frame:, reference_images:, reference_videos:, reference_audios:)
|
|
261
|
+
has_frame = !normalize_list(first_frame).empty? || !normalize_list(last_frame).empty?
|
|
262
|
+
has_reference = !normalize_list(reference_images).empty? ||
|
|
263
|
+
!normalize_list(reference_videos).empty? ||
|
|
264
|
+
!normalize_list(reference_audios).empty?
|
|
265
|
+
if has_frame && has_reference
|
|
266
|
+
return [nil, "first_frame/last_frame cannot be combined with reference_images/videos/audios. " \
|
|
267
|
+
"Use first_frame (and optionally last_frame) for first/last-frame image-to-video, " \
|
|
268
|
+
"OR use reference_* for multimodal / video editing / video extension — not both."]
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
content = []
|
|
272
|
+
content << { type: "text", text: prompt.to_s } unless prompt.to_s.strip.empty?
|
|
273
|
+
|
|
274
|
+
{
|
|
275
|
+
"first_frame" => normalize_list(first_frame),
|
|
276
|
+
"last_frame" => normalize_list(last_frame),
|
|
277
|
+
"reference_image" => normalize_list(reference_images)
|
|
278
|
+
}.each do |role, items|
|
|
279
|
+
items.each do |item|
|
|
280
|
+
url, err = to_media_url(item, kind: :image)
|
|
281
|
+
return [nil, err] if err
|
|
282
|
+
content << { type: "image_url", image_url: { url: url }, role: role }
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
normalize_list(reference_videos).each do |item|
|
|
287
|
+
url, err = to_media_url(item, kind: :video)
|
|
288
|
+
return [nil, err] if err
|
|
289
|
+
content << { type: "video_url", video_url: { url: url }, role: "reference_video" }
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
normalize_list(reference_audios).each do |item|
|
|
293
|
+
url, err = to_media_url(item, kind: :audio)
|
|
294
|
+
return [nil, err] if err
|
|
295
|
+
content << { type: "audio_url", audio_url: { url: url }, role: "reference_audio" }
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
[content, nil]
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Accept a single value or an array; drop nils/blanks.
|
|
302
|
+
private def normalize_list(value)
|
|
303
|
+
list = value.is_a?(Array) ? value : [value]
|
|
304
|
+
list.reject { |v| v.nil? || (v.is_a?(String) && v.strip.empty?) }
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Resolve one media item into a URL Ark accepts (http(s) URL or data
|
|
308
|
+
# URL). Accepts: an http(s)/data URL string, a local file path, or a
|
|
309
|
+
# { "b64_json" => ..., "mime_type" => ... } hash. Returns [url, nil] or
|
|
310
|
+
# [nil, error_message].
|
|
311
|
+
private def to_media_url(item, kind:)
|
|
312
|
+
if item.is_a?(Hash)
|
|
313
|
+
b64 = item["b64_json"] || item[:b64_json]
|
|
314
|
+
mime = (item["mime_type"] || item[:mime_type]).to_s
|
|
315
|
+
return [nil, "media hash is missing b64_json"] if b64.to_s.empty?
|
|
316
|
+
mime = default_mime(kind) if mime.empty?
|
|
317
|
+
return ["data:#{mime};base64,#{b64}", nil]
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
s = item.to_s.strip
|
|
321
|
+
return [nil, "empty media reference"] if s.empty?
|
|
322
|
+
return [s, nil] if s.start_with?("http://", "https://", "data:")
|
|
323
|
+
|
|
324
|
+
unless File.file?(s)
|
|
325
|
+
return [nil, "media reference is neither an existing file path, a URL, nor base64 data: #{s}"]
|
|
326
|
+
end
|
|
327
|
+
bytes = File.binread(s)
|
|
328
|
+
mime = mime_for_path(s, kind)
|
|
329
|
+
["data:#{mime};base64,#{Base64.strict_encode64(bytes)}", nil]
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
private def resolve_ratio(aspect_ratio)
|
|
333
|
+
s = aspect_ratio.to_s
|
|
334
|
+
return s if ARK_RATIOS.include?(s)
|
|
335
|
+
ASPECT_TO_RATIO[s] || DEFAULT_RATIO
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
private def default_mime(kind)
|
|
339
|
+
case kind
|
|
340
|
+
when :video then "video/mp4"
|
|
341
|
+
when :audio then "audio/mpeg"
|
|
342
|
+
else "image/png"
|
|
343
|
+
end
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
private def mime_for_path(path, kind)
|
|
347
|
+
case File.extname(path).downcase
|
|
348
|
+
when ".jpg", ".jpeg" then "image/jpeg"
|
|
349
|
+
when ".webp" then "image/webp"
|
|
350
|
+
when ".gif" then "image/gif"
|
|
351
|
+
when ".mp4" then "video/mp4"
|
|
352
|
+
when ".mov" then "video/quicktime"
|
|
353
|
+
when ".webm" then "video/webm"
|
|
354
|
+
when ".mp3" then "audio/mpeg"
|
|
355
|
+
when ".wav" then "audio/wav"
|
|
356
|
+
when ".m4a" then "audio/mp4"
|
|
357
|
+
else default_mime(kind)
|
|
358
|
+
end
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
private def connection
|
|
362
|
+
Faraday.new(url: endpoint_base) do |f|
|
|
363
|
+
f.options.timeout = 240
|
|
364
|
+
f.options.open_timeout = 10
|
|
365
|
+
end
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
# Derive the API root (scheme + host) from the configured base_url,
|
|
369
|
+
# discarding any path the user pasted (e.g. /api/v3). The task path is
|
|
370
|
+
# appended by the request methods. Falls back to the Beijing host.
|
|
371
|
+
private def endpoint_base
|
|
372
|
+
uri = URI.parse(@base_url.to_s)
|
|
373
|
+
if uri.scheme && uri.host
|
|
374
|
+
"#{uri.scheme}://#{uri.host}"
|
|
375
|
+
else
|
|
376
|
+
"https://ark.cn-beijing.volces.com"
|
|
377
|
+
end
|
|
378
|
+
rescue URI::InvalidURIError
|
|
379
|
+
"https://ark.cn-beijing.volces.com"
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
private def parse_json(body)
|
|
383
|
+
JSON.parse(body)
|
|
384
|
+
rescue JSON::ParserError
|
|
385
|
+
nil
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
private def truncate(str, max)
|
|
389
|
+
s = str.to_s
|
|
390
|
+
s.length > max ? "#{s[0, max]}..." : s
|
|
391
|
+
end
|
|
392
|
+
end
|
|
393
|
+
end
|
|
394
|
+
end
|
|
@@ -53,7 +53,7 @@ module Clacky
|
|
|
53
53
|
# encode + downscale on every history replay (2-3s lag for sessions
|
|
54
54
|
# with downgraded text-model images). The proxy lets the browser
|
|
55
55
|
# lazy-load + cache the image, keeping the replay response tiny.
|
|
56
|
-
"/api/local-image?path=#{CGI.escape(path.to_s)}"
|
|
56
|
+
"/api/local-image?path=#{CGI.escape(path.to_s)}&v=#{File.mtime(path.to_s).to_i}"
|
|
57
57
|
elsif name
|
|
58
58
|
type.to_s == "image" ? "expired:#{name}" : "pdf:#{name}"
|
|
59
59
|
end
|
|
@@ -463,6 +463,9 @@ module Clacky
|
|
|
463
463
|
# with modalities:["image"]); end-to-end latency is commonly
|
|
464
464
|
# 20-60s and can exceed 2 minutes for or-gpt-image-2 under load.
|
|
465
465
|
300
|
|
466
|
+
elsif path == "/api/media/video/status"
|
|
467
|
+
# Single upstream task lookup; returns in well under a second.
|
|
468
|
+
30
|
|
466
469
|
elsif path == "/api/media/video"
|
|
467
470
|
# Video generation (Veo via the gateway) runs an async submit+poll
|
|
468
471
|
# cycle that routinely takes 1-3 minutes and can approach the
|
|
@@ -572,6 +575,7 @@ module Clacky
|
|
|
572
575
|
when ["GET", "/api/local-image"] then api_serve_local_image(req, res)
|
|
573
576
|
when ["POST", "/api/media/image"] then api_media_image(req, res)
|
|
574
577
|
when ["POST", "/api/media/video"] then api_media_video(req, res)
|
|
578
|
+
when ["GET", "/api/media/video/status"] then api_media_video_status(req, res)
|
|
575
579
|
when ["POST", "/api/media/audio/speech"] then api_media_audio_speech(req, res)
|
|
576
580
|
when ["POST", "/api/media/audio/transcriptions"] then api_media_audio_transcriptions(req, res)
|
|
577
581
|
when ["POST", "/api/media/video/understand"] then api_media_video_understand(req, res)
|
|
@@ -1539,7 +1543,16 @@ module Clacky
|
|
|
1539
1543
|
aspect_ratio: aspect_ratio,
|
|
1540
1544
|
duration_seconds: duration,
|
|
1541
1545
|
output_dir: output_dir,
|
|
1542
|
-
image: image
|
|
1546
|
+
image: image,
|
|
1547
|
+
first_frame: body["first_frame"],
|
|
1548
|
+
last_frame: body["last_frame"],
|
|
1549
|
+
reference_images: body["reference_images"],
|
|
1550
|
+
reference_videos: body["reference_videos"],
|
|
1551
|
+
reference_audios: body["reference_audios"],
|
|
1552
|
+
resolution: body["resolution"],
|
|
1553
|
+
generate_audio: body["generate_audio"],
|
|
1554
|
+
watermark: body["watermark"],
|
|
1555
|
+
seed: body["seed"]
|
|
1543
1556
|
)
|
|
1544
1557
|
if result["success"]
|
|
1545
1558
|
log_media_usage(result, prompt: prompt, session_id: session_id)
|
|
@@ -1550,6 +1563,25 @@ module Clacky
|
|
|
1550
1563
|
json_response(res, 500, { error: e.message })
|
|
1551
1564
|
end
|
|
1552
1565
|
|
|
1566
|
+
def api_media_video_status(req, res)
|
|
1567
|
+
query = URI.decode_www_form(req.query_string.to_s).to_h
|
|
1568
|
+
task_id = query["task_id"].to_s
|
|
1569
|
+
if task_id.strip.empty?
|
|
1570
|
+
return json_response(res, 422, { error: "task_id is required" })
|
|
1571
|
+
end
|
|
1572
|
+
|
|
1573
|
+
output_dir = query["output_dir"].to_s
|
|
1574
|
+
output_dir = @agent_config.default_working_dir || Dir.pwd if output_dir.empty?
|
|
1575
|
+
|
|
1576
|
+
result = Clacky::Media::Generator.new(@agent_config).video_status(
|
|
1577
|
+
task_id: task_id,
|
|
1578
|
+
output_dir: output_dir
|
|
1579
|
+
)
|
|
1580
|
+
json_response(res, 200, result)
|
|
1581
|
+
rescue StandardError => e
|
|
1582
|
+
json_response(res, 500, { error: e.message })
|
|
1583
|
+
end
|
|
1584
|
+
|
|
1553
1585
|
def api_media_audio_speech(req, res)
|
|
1554
1586
|
body = parse_json_body(req)
|
|
1555
1587
|
return json_response(res, 400, { error: "Invalid JSON" }) unless body
|
|
@@ -3730,6 +3762,20 @@ module Clacky
|
|
|
3730
3762
|
file_size = File.size(path)
|
|
3731
3763
|
mime = Utils::FileProcessor::MIME_TYPES[ext] || "application/octet-stream"
|
|
3732
3764
|
|
|
3765
|
+
# ETag from mtime+size so an overwritten same-name file invalidates the
|
|
3766
|
+
# browser cache. Use no-cache (revalidate every time) rather than
|
|
3767
|
+
# max-age: unchanged files return 304 (no body), changed files return
|
|
3768
|
+
# the new bytes.
|
|
3769
|
+
stat = File.stat(path)
|
|
3770
|
+
etag = %(W/"#{stat.mtime.to_i}-#{stat.size}")
|
|
3771
|
+
res["Cache-Control"] = "private, no-cache"
|
|
3772
|
+
res["ETag"] = etag
|
|
3773
|
+
if req["If-None-Match"] == etag
|
|
3774
|
+
res.status = 304
|
|
3775
|
+
res.body = ""
|
|
3776
|
+
return
|
|
3777
|
+
end
|
|
3778
|
+
|
|
3733
3779
|
# Support HTTP Range requests for video seeking
|
|
3734
3780
|
range_header = req["Range"]
|
|
3735
3781
|
if range_header && range_header =~ /\Abytes=(\d*)-(\d*)\z/
|
|
@@ -3741,14 +3787,12 @@ module Clacky
|
|
|
3741
3787
|
res["Content-Type"] = mime
|
|
3742
3788
|
res["Content-Range"] = "bytes #{start_byte}-#{end_byte}/#{file_size}"
|
|
3743
3789
|
res["Accept-Ranges"] = "bytes"
|
|
3744
|
-
res["Cache-Control"] = "private, max-age=3600"
|
|
3745
3790
|
res["Content-Length"] = (end_byte - start_byte + 1).to_s
|
|
3746
3791
|
IO.binread(path, end_byte - start_byte + 1, start_byte).then { |data| res.body = data }
|
|
3747
3792
|
else
|
|
3748
3793
|
res.status = 200
|
|
3749
3794
|
res["Content-Type"] = mime
|
|
3750
3795
|
res["Accept-Ranges"] = "bytes"
|
|
3751
|
-
res["Cache-Control"] = "private, max-age=3600"
|
|
3752
3796
|
res.body = File.binread(path)
|
|
3753
3797
|
end
|
|
3754
3798
|
rescue => e
|
|
@@ -607,9 +607,22 @@ module Clacky
|
|
|
607
607
|
""
|
|
608
608
|
)
|
|
609
609
|
|
|
610
|
-
# Pass 1: anchored strip — the
|
|
611
|
-
#
|
|
612
|
-
|
|
610
|
+
# Pass 1: anchored strip — remove ONLY the wrapper's echoed head
|
|
611
|
+
# line (`{ USER_CMD`) at the very start of the buffer, and ONLY
|
|
612
|
+
# when a matching wrapper tail fingerprint (`}...; __clacky_ec=$?`)
|
|
613
|
+
# actually exists somewhere in the buffer. The guard is what keeps
|
|
614
|
+
# us from eating a real first line of user output that merely
|
|
615
|
+
# happens to start with `{` (e.g. JSON like `{"key": ...}`).
|
|
616
|
+
# We test the tail with a separate anchor-free `match?` (linear, no
|
|
617
|
+
# backtracking) rather than a lookahead, to avoid catastrophic
|
|
618
|
+
# regex backtracking on large `{`-prefixed output. We strip a
|
|
619
|
+
# single line (`[^\n]*`, not cross-line `.*?`): a multi-line command
|
|
620
|
+
# echoed in cooked mode puts the wrapper tail *after* the real
|
|
621
|
+
# output, so cross-line greed would swallow that output whole. The
|
|
622
|
+
# tail itself is removed by the fingerprint passes below.
|
|
623
|
+
if text.start_with?("{") && text.match?(/\}[^\n;]*; *__clacky_ec=\$\?/)
|
|
624
|
+
text = text.sub(/\A\{[^\n]*\n/, "")
|
|
625
|
+
end
|
|
613
626
|
|
|
614
627
|
# Pass 2: token-aware global strip — remove any leftover wrapper
|
|
615
628
|
# echo fragment, wherever it sits. Requires the session token so
|
|
@@ -629,9 +642,11 @@ module Clacky
|
|
|
629
642
|
# 2b. Single-line shape: everything collapsed onto one line.
|
|
630
643
|
# Strip from the wrapper's `}; __clacky_ec=$?` pivot (or the
|
|
631
644
|
# opening `{` if still present on that line) through the end of
|
|
632
|
-
# the printf invocation (`"$__clacky_ec"`).
|
|
645
|
+
# the printf invocation (`"$__clacky_ec"`). The optional
|
|
646
|
+
# `[^\n;]*` between `}` and `;` absorbs a WSL `</dev/null`
|
|
647
|
+
# stdin redirect spliced onto the group.
|
|
633
648
|
text = text.gsub(
|
|
634
|
-
/[^\n]*\}; *__clacky_ec=\$\?; *printf[^\n]*__CLACKY_DONE_#{token_re}_[^\n]*"\$__clacky_ec"[^\n]*\n?/,
|
|
649
|
+
/[^\n]*\}[^\n;]*; *__clacky_ec=\$\?; *printf[^\n]*__CLACKY_DONE_#{token_re}_[^\n]*"\$__clacky_ec"[^\n]*\n?/,
|
|
635
650
|
""
|
|
636
651
|
)
|
|
637
652
|
|
|
@@ -653,9 +668,10 @@ module Clacky
|
|
|
653
668
|
# private double-underscore var name that user code effectively
|
|
654
669
|
# never emits — so we strip anything between them (non-greedy,
|
|
655
670
|
# multiline-aware) to also handle width-wrap that inserted
|
|
656
|
-
# real \n breaks inside the echo.
|
|
671
|
+
# real \n breaks inside the echo. `[^\n;]*` between `}` and `;`
|
|
672
|
+
# absorbs a WSL `</dev/null` stdin redirect on the group.
|
|
657
673
|
text = text.gsub(
|
|
658
|
-
/[^\n]*\}; *__clacky_ec=\$\?.*?"\$__clacky_ec"[^\n]*\n?/m,
|
|
674
|
+
/[^\n]*\}[^\n;]*; *__clacky_ec=\$\?.*?"\$__clacky_ec"[^\n]*\n?/m,
|
|
659
675
|
""
|
|
660
676
|
)
|
|
661
677
|
|
|
@@ -663,7 +679,7 @@ module Clacky
|
|
|
663
679
|
# truncation cut off everything after `__clacky_ec=$?`). Still a
|
|
664
680
|
# reliable fingerprint thanks to the `__clacky_ec` var name.
|
|
665
681
|
text = text.gsub(
|
|
666
|
-
/[^\n]*\}; *__clacky_ec=\$\?;?[^\n]*\n?/,
|
|
682
|
+
/[^\n]*\}[^\n;]*; *__clacky_ec=\$\?;?[^\n]*\n?/,
|
|
667
683
|
""
|
|
668
684
|
)
|
|
669
685
|
|
|
@@ -402,6 +402,10 @@ module Clacky
|
|
|
402
402
|
screen.show_cursor
|
|
403
403
|
screen.flush
|
|
404
404
|
end
|
|
405
|
+
# Restore cooked mode so the terminal is usable after exit, regardless
|
|
406
|
+
# of which exit path was taken (some call exit(0) and bypass the
|
|
407
|
+
# input_loop ensure). Idempotent; safe to call more than once.
|
|
408
|
+
screen.disable_raw_mode
|
|
405
409
|
end
|
|
406
410
|
|
|
407
411
|
# /clear: wipe output area + buffer, keep fixed area.
|
|
@@ -118,14 +118,42 @@ module Clacky
|
|
|
118
118
|
@height = TTY::Screen.height
|
|
119
119
|
end
|
|
120
120
|
|
|
121
|
-
# Enable raw mode (disable line buffering)
|
|
121
|
+
# Enable raw mode (disable line buffering). The very first time it runs it
|
|
122
|
+
# snapshots the terminal's original settings (via `stty -g`) so the exact
|
|
123
|
+
# pre-clacky state can be restored on exit — `IO#cooked!` only applies a
|
|
124
|
+
# generic cooked profile and does not restore flow-control (IXON) and other
|
|
125
|
+
# bits, which on macOS leaves the shell in a broken input state.
|
|
122
126
|
def enable_raw_mode
|
|
127
|
+
return unless $stdin.tty?
|
|
128
|
+
|
|
129
|
+
@original_stty ||= capture_stty
|
|
123
130
|
$stdin.raw!
|
|
124
131
|
end
|
|
125
132
|
|
|
126
|
-
# Disable raw mode
|
|
133
|
+
# Disable raw mode. Idempotent and tty-guarded so it can be safely called
|
|
134
|
+
# from any exit path. Restores the exact original terminal settings when a
|
|
135
|
+
# snapshot exists, then flushes pending input so stray bytes don't leak to
|
|
136
|
+
# the shell. Falls back to `IO#cooked!` when no snapshot is available.
|
|
127
137
|
def disable_raw_mode
|
|
128
|
-
$stdin.
|
|
138
|
+
return unless $stdin.tty?
|
|
139
|
+
|
|
140
|
+
$stdin.cooked! unless @original_stty && restore_stty(@original_stty)
|
|
141
|
+
$stdin.ioflush
|
|
142
|
+
rescue IOError, Errno::ENOTTY
|
|
143
|
+
nil
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
private def capture_stty
|
|
147
|
+
out = Clacky::Utils::Encoding.cmd_to_utf8(`stty -g 2>/dev/null`).strip
|
|
148
|
+
out.empty? ? nil : out
|
|
149
|
+
rescue StandardError
|
|
150
|
+
nil
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
private def restore_stty(state)
|
|
154
|
+
system("stty", state, out: File::NULL, err: File::NULL)
|
|
155
|
+
rescue StandardError
|
|
156
|
+
false
|
|
129
157
|
end
|
|
130
158
|
|
|
131
159
|
# Read a single character without echo
|
|
@@ -604,8 +604,7 @@ module Clacky
|
|
|
604
604
|
|
|
605
605
|
ext = File.extname(path).downcase
|
|
606
606
|
if LOCAL_MEDIA_EXTENSIONS.include?(ext) && File.exist?(path)
|
|
607
|
-
|
|
608
|
-
""
|
|
607
|
+
"})"
|
|
609
608
|
else
|
|
610
609
|
_match
|
|
611
610
|
end
|
|
@@ -622,8 +621,7 @@ module Clacky
|
|
|
622
621
|
|
|
623
622
|
ext = File.extname(path).downcase
|
|
624
623
|
if LOCAL_VIDEO_EXTENSIONS.include?(ext) && File.exist?(path)
|
|
625
|
-
|
|
626
|
-
"<video#{pre} src=\"/api/local-image?path=#{encoded}\"#{post}>"
|
|
624
|
+
"<video#{pre} src=\"#{local_image_proxy_url(href, path)}\"#{post}>"
|
|
627
625
|
else
|
|
628
626
|
_match
|
|
629
627
|
end
|
|
@@ -640,8 +638,7 @@ module Clacky
|
|
|
640
638
|
|
|
641
639
|
ext = File.extname(path).downcase
|
|
642
640
|
if LOCAL_AUDIO_EXTENSIONS.include?(ext) && File.exist?(path)
|
|
643
|
-
|
|
644
|
-
"<audio#{pre} src=\"/api/local-image?path=#{encoded}\"#{post}>"
|
|
641
|
+
"<audio#{pre} src=\"#{local_image_proxy_url(href, path)}\"#{post}>"
|
|
645
642
|
else
|
|
646
643
|
_match
|
|
647
644
|
end
|
|
@@ -658,8 +655,7 @@ module Clacky
|
|
|
658
655
|
ext = File.extname(path).downcase
|
|
659
656
|
if LOCAL_VIDEO_EXTENSIONS.include?(ext) || LOCAL_AUDIO_EXTENSIONS.include?(ext)
|
|
660
657
|
if File.exist?(path)
|
|
661
|
-
|
|
662
|
-
"[#{text}](/api/local-image?path=#{encoded})"
|
|
658
|
+
"[#{text}](#{local_image_proxy_url(href, path)})"
|
|
663
659
|
else
|
|
664
660
|
_match
|
|
665
661
|
end
|
|
@@ -670,6 +666,15 @@ module Clacky
|
|
|
670
666
|
|
|
671
667
|
content
|
|
672
668
|
end
|
|
669
|
+
|
|
670
|
+
# Build a proxy URL for a local media file, appending a version param
|
|
671
|
+
# derived from the file mtime so an overwritten same-name file produces a
|
|
672
|
+
# different URL and defeats the browser's in-memory image cache.
|
|
673
|
+
def self.local_image_proxy_url(href, path)
|
|
674
|
+
encoded = CGI.escape(href)
|
|
675
|
+
version = File.mtime(path).to_i
|
|
676
|
+
"/api/local-image?path=#{encoded}&v=#{version}"
|
|
677
|
+
end
|
|
673
678
|
end
|
|
674
679
|
end
|
|
675
680
|
end
|