cohere-transcribe 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +9 -7
- data/ext/cohere_transcribe_native/.gitignore +6 -0
- data/ext/cohere_transcribe_native/CMakeLists.txt +8 -0
- data/ext/cohere_transcribe_native/audio_abi.cpp +20 -0
- data/ext/cohere_transcribe_native/audio_exports.macos +1 -0
- data/ext/cohere_transcribe_native/audio_exports.map +1 -0
- data/ext/cohere_transcribe_native/test/abi_smoke.rb +26 -0
- data/lib/cohere/transcribe/alignment/aligner.rb +40 -7
- data/lib/cohere/transcribe/asr/native.rb +43 -97
- data/lib/cohere/transcribe/audio/decoder.rb +184 -75
- data/lib/cohere/transcribe/audio/ffmpeg_native.rb +84 -32
- data/lib/cohere/transcribe/audio/segmentation.rb +1 -0
- data/lib/cohere/transcribe/cli.rb +30 -22
- data/lib/cohere/transcribe/configuration.rb +13 -0
- data/lib/cohere/transcribe/constants.rb +1 -1
- data/lib/cohere/transcribe/doctor.rb +2 -1
- data/lib/cohere/transcribe/hub.rb +446 -58
- data/lib/cohere/transcribe/input.rb +18 -3
- data/lib/cohere/transcribe/internal/interruptible_native_call.rb +70 -0
- data/lib/cohere/transcribe/internal/session_ownership.rb +61 -0
- data/lib/cohere/transcribe/internal/utf8.rb +19 -0
- data/lib/cohere/transcribe/model_identity.rb +10 -1
- data/lib/cohere/transcribe/output/publication.rb +95 -86
- data/lib/cohere/transcribe/pytorch_checkpoint.rb +38 -22
- data/lib/cohere/transcribe/runtime/engine.rb +37 -16
- data/lib/cohere/transcribe/runtime/preparation.rb +212 -35
- data/lib/cohere/transcribe/runtime/resources.rb +28 -62
- data/lib/cohere/transcribe/state/checkpoint.rb +3 -2
- data/lib/cohere/transcribe/state/io.rb +217 -117
- data/lib/cohere/transcribe/state/locking.rb +383 -64
- data/lib/cohere/transcribe/state/manifest.rb +21 -22
- data/lib/cohere/transcribe/types.rb +25 -11
- data/lib/cohere/transcribe/version.rb +1 -1
- data/sig/cohere/transcribe.rbs +1 -0
- metadata +5 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "optparse"
|
|
4
|
+
require_relative "constants"
|
|
4
5
|
require_relative "errors"
|
|
5
6
|
require_relative "version"
|
|
6
7
|
|
|
@@ -8,8 +9,8 @@ module Cohere
|
|
|
8
9
|
module Transcribe
|
|
9
10
|
# Command-line adapter for the dependency-light public API.
|
|
10
11
|
module CLI
|
|
11
|
-
DEFAULT_MODEL_ID =
|
|
12
|
-
OUTPUT_FORMATS =
|
|
12
|
+
DEFAULT_MODEL_ID = DEFAULT_ASR_MODEL_ID
|
|
13
|
+
OUTPUT_FORMATS = Cohere::Transcribe::OUTPUT_FORMATS
|
|
13
14
|
OUTPUT_PATH_DISPLAY_LIMIT = 20
|
|
14
15
|
FORMAT_ARGUMENT_SEPARATOR = "\u001F"
|
|
15
16
|
private_constant :FORMAT_ARGUMENT_SEPARATOR
|
|
@@ -179,8 +180,8 @@ module Cohere
|
|
|
179
180
|
parser.on("--adapter-revision REVISION", "Hub adapter commit, tag, or branch.") do |value|
|
|
180
181
|
values[:adapter_revision] = value
|
|
181
182
|
end
|
|
182
|
-
parser.on("--language LANGUAGE", String,
|
|
183
|
-
values[:language] = value
|
|
183
|
+
parser.on("--language LANGUAGE", String, "Spoken language tag (ar or en).") do |value|
|
|
184
|
+
values[:language] = exact_choice!("language", value, %w[ar en])
|
|
184
185
|
end
|
|
185
186
|
parser.on("--formats FORMAT [FORMAT ...]", String, "Outputs to write: txt, srt, vtt, json.") do |encoded|
|
|
186
187
|
value = encoded.split(FORMAT_ARGUMENT_SEPARATOR, -1)
|
|
@@ -196,21 +197,20 @@ module Cohere
|
|
|
196
197
|
end
|
|
197
198
|
parser.on("--recursive", "Recurse into input directories (default).") { values[:recursive] = true }
|
|
198
199
|
parser.on("--no-recursive", "Do not recurse into input directories.") { values[:recursive] = false }
|
|
199
|
-
parser.on("--existing POLICY", String,
|
|
200
|
-
values[:existing] = value
|
|
200
|
+
parser.on("--existing POLICY", String, "Existing outputs policy.") do |value|
|
|
201
|
+
values[:existing] = exact_choice!("existing", value, %w[error overwrite skip])
|
|
201
202
|
end
|
|
202
|
-
parser.on("--device DEVICE", String,
|
|
203
|
-
values[:device] = value
|
|
203
|
+
parser.on("--device DEVICE", String, "Inference device.") do |value|
|
|
204
|
+
values[:device] = exact_choice!("device", value, %w[auto mps cuda cpu])
|
|
204
205
|
end
|
|
205
|
-
parser.on("--dtype DTYPE", String,
|
|
206
|
-
values[:dtype] = value
|
|
206
|
+
parser.on("--dtype DTYPE", String, "ASR model precision.") do |value|
|
|
207
|
+
values[:dtype] = exact_choice!("dtype", value, %w[auto bf16 fp16 fp32])
|
|
207
208
|
end
|
|
208
209
|
parser.on(
|
|
209
210
|
"--audio-backend BACKEND",
|
|
210
211
|
String,
|
|
211
|
-
%w[auto torchcodec ffmpeg librosa],
|
|
212
212
|
"Audio decoder configuration."
|
|
213
|
-
) { |value| values[:audio_backend] = value }
|
|
213
|
+
) { |value| values[:audio_backend] = exact_choice!("audio-backend", value, %w[auto torchcodec ffmpeg librosa]) }
|
|
214
214
|
parser.on("--audio-memory-gb GIB", String, "Decoded-PCM memory limit per file/group.") do |value|
|
|
215
215
|
values[:audio_memory_gb] = parse_reference_float(value)
|
|
216
216
|
end
|
|
@@ -226,11 +226,11 @@ module Cohere
|
|
|
226
226
|
|
|
227
227
|
parser.separator("")
|
|
228
228
|
parser.separator("Segmentation (VAD):")
|
|
229
|
-
parser.on("--vad MODE", String,
|
|
230
|
-
values[:vad] = value
|
|
229
|
+
parser.on("--vad MODE", String, "Segmentation policy.") do |value|
|
|
230
|
+
values[:vad] = exact_choice!("vad", value, %w[silero auditok none])
|
|
231
231
|
end
|
|
232
|
-
parser.on("--vad-engine ENGINE", String,
|
|
233
|
-
values[:vad_engine] = value
|
|
232
|
+
parser.on("--vad-engine ENGINE", String, "Silero runtime.") do |value|
|
|
233
|
+
values[:vad_engine] = exact_choice!("vad-engine", value, %w[auto torch onnx jit])
|
|
234
234
|
end
|
|
235
235
|
parser.on("--vad-batch-size COUNT", String, "Maximum files per packed VAD call.") do |value|
|
|
236
236
|
values[:vad_batch_size] = parse_reference_integer(value)
|
|
@@ -299,8 +299,8 @@ module Cohere
|
|
|
299
299
|
parser.on("--max-retry-tokens COUNT", String, "Automatic retry token limit.") do |value|
|
|
300
300
|
values[:max_retry_tokens] = parse_reference_integer(value)
|
|
301
301
|
end
|
|
302
|
-
parser.on("--truncation-policy POLICY", String,
|
|
303
|
-
values[:truncation_policy] = value
|
|
302
|
+
parser.on("--truncation-policy POLICY", String, "Token-limit behavior.") do |value|
|
|
303
|
+
values[:truncation_policy] = exact_choice!("truncation-policy", value, %w[retry warn])
|
|
304
304
|
end
|
|
305
305
|
parser.on("--stop-repetition-loops", "Stop conservative decoder repetition loops (default).") do
|
|
306
306
|
values[:stop_repetition_loops] = true
|
|
@@ -311,9 +311,9 @@ module Cohere
|
|
|
311
311
|
|
|
312
312
|
parser.separator("")
|
|
313
313
|
parser.separator("Alignment and subtitle cues:")
|
|
314
|
-
parser.on("--alignment MODE", String,
|
|
314
|
+
parser.on("--alignment MODE", String, "Timestamp mode.") do |value|
|
|
315
315
|
state[:alignment] = true
|
|
316
|
-
values[:alignment] = value
|
|
316
|
+
values[:alignment] = exact_choice!("alignment", value, %w[word segment none])
|
|
317
317
|
end
|
|
318
318
|
parser.on("--text-only", "Alias for --alignment none.") do
|
|
319
319
|
state[:text_only] = true
|
|
@@ -322,8 +322,8 @@ module Cohere
|
|
|
322
322
|
parser.on("--align-batch-size COUNT", String, "Maximum alignment windows per batch.") do |value|
|
|
323
323
|
values[:align_batch_size] = parse_reference_integer(value)
|
|
324
324
|
end
|
|
325
|
-
parser.on("--align-dtype DTYPE", String,
|
|
326
|
-
values[:align_dtype] = value
|
|
325
|
+
parser.on("--align-dtype DTYPE", String, "Alignment precision.") do |value|
|
|
326
|
+
values[:align_dtype] = exact_choice!("align-dtype", value, %w[fp32 fp16])
|
|
327
327
|
end
|
|
328
328
|
parser.on("--max-chars COUNT", String, "Target subtitle cue length.") do |value|
|
|
329
329
|
values[:max_chars] = parse_reference_integer(value)
|
|
@@ -557,6 +557,14 @@ module Cohere
|
|
|
557
557
|
end
|
|
558
558
|
private_class_method :defer_unknown_options_before_early_exit
|
|
559
559
|
|
|
560
|
+
def exact_choice!(option, value, choices)
|
|
561
|
+
return value if choices.include?(value)
|
|
562
|
+
|
|
563
|
+
raise OptionParser::InvalidArgument,
|
|
564
|
+
"--#{option}: invalid choice #{value.inspect}; choose from #{choices.join(", ")}"
|
|
565
|
+
end
|
|
566
|
+
private_class_method :exact_choice!
|
|
567
|
+
|
|
560
568
|
def parse_reference_float(value)
|
|
561
569
|
text = normalize_reference_numeric_text(value)
|
|
562
570
|
special = text.match?(/\A[+-]?(?:inf(?:inity)?|nan)\z/i)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "constants"
|
|
4
4
|
require_relative "errors"
|
|
5
|
+
require_relative "internal/utf8"
|
|
5
6
|
require_relative "model_identity"
|
|
6
7
|
require_relative "python_text"
|
|
7
8
|
require_relative "types"
|
|
@@ -200,6 +201,7 @@ module Cohere
|
|
|
200
201
|
text = value.is_a?(String) ? value : value.to_path
|
|
201
202
|
raise TypeError, "#{description} path must resolve to text" unless text.is_a?(String)
|
|
202
203
|
|
|
204
|
+
text = validated_utf8_text(text, option)
|
|
203
205
|
stripped = PythonText.strip(text)
|
|
204
206
|
invalid!("#{option} must be a non-empty Hugging Face repository ID or local directory") if stripped.empty?
|
|
205
207
|
invalid!("#{option} must not have leading or trailing whitespace") unless text == stripped
|
|
@@ -228,6 +230,7 @@ module Cohere
|
|
|
228
230
|
return if value.nil?
|
|
229
231
|
|
|
230
232
|
if value.is_a?(String)
|
|
233
|
+
value = validated_utf8_text(value, option)
|
|
231
234
|
stripped = PythonText.strip(value)
|
|
232
235
|
return if !stripped.empty? && value == stripped
|
|
233
236
|
end
|
|
@@ -235,6 +238,16 @@ module Cohere
|
|
|
235
238
|
invalid!("#{option} must be a non-empty revision without surrounding whitespace")
|
|
236
239
|
end
|
|
237
240
|
|
|
241
|
+
private_class_method :validate_revision!
|
|
242
|
+
|
|
243
|
+
def validated_utf8_text(value, option)
|
|
244
|
+
text = Internal::UTF8.normalize(value)
|
|
245
|
+
invalid!("#{option} must contain valid UTF-8") unless text
|
|
246
|
+
|
|
247
|
+
text
|
|
248
|
+
end
|
|
249
|
+
private_class_method :validated_utf8_text
|
|
250
|
+
|
|
238
251
|
def valid_repository_id?(value)
|
|
239
252
|
return false if value.count("/") > 1 || value.include?("--") || value.include?("..") || value.end_with?(".git")
|
|
240
253
|
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "optparse"
|
|
4
|
+
require_relative "constants"
|
|
4
5
|
require_relative "version"
|
|
5
6
|
|
|
6
7
|
module Cohere
|
|
7
8
|
module Transcribe
|
|
8
9
|
# Installation and model-metadata diagnostics that never load ASR weights.
|
|
9
10
|
module Doctor
|
|
10
|
-
DEFAULT_MODEL_ID =
|
|
11
|
+
DEFAULT_MODEL_ID = DEFAULT_ASR_MODEL_ID
|
|
11
12
|
EXPECTED_ONNX_SHA256 = "914fd98ac0a73d69ba1e70c9b1d66acb740eff90500dfde08b89a961b168a6a9"
|
|
12
13
|
COHERE_PROCESSOR_CLASS = "CohereAsrProcessor"
|
|
13
14
|
COHERE_FEATURE_EXTRACTOR_CLASS = "CohereAsrFeatureExtractor"
|