substrate 0.1.0 → 0.3.0

data/lib/substrate/transcribers/whisper_cpp.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "json"
+
+module Substrate
+  module Transcribers
+    class WhisperCpp < Command
+      class TranscriptionError < StandardError; end
+
+      DEFAULT_MODEL = "base"
+      DEFAULT_THREADS = 4
+
+      def initialize(path:, model: DEFAULT_MODEL, threads: DEFAULT_THREADS, language: nil, model_path: "~/.local/share/whisper/models/ggml-#{model}.bin", executable: :auto)
+        @path = path
+        @threads = threads
+        @language = language
+        @model_path = File.expand_path(model_path || "~/.local/share/whisper/models/ggml-#{model}.bin")
+        @executable = executable || :auto
+      end
+
+      def call
+        select_transcription_executable
+        verify_transcription_requirements
+        transcribe_recording
+      rescue ArgumentError => error
+        failure(error)
+      end
+
+      private
+
+      attr_reader :path, :threads, :language, :model_path, :executable
+
+      def select_transcription_executable
+        @executable = (Cli::Availability.call("whisper-cli") ? "whisper-cli" : "whisper") if executable == :auto
+      end
+
+      def verify_transcription_requirements
+        unless Cli::Availability.call(executable)
+          raise ArgumentError, "Whisper CLI not found: #{executable}. Install whisper.cpp or pass executable:"
+        end
+        unless File.file?(model_path) && File.readable?(model_path)
+          raise ArgumentError, "Whisper model is not readable: #{model_path}"
+        end
+      end
+
+      def transcribe_recording
+        transcript = Dir.mktmpdir("substrate-whisper-") do |directory|
+          audio_path = File.join(directory, "audio.wav")
+          output_prefix = File.join(directory, "transcript")
+          run!(preparation_arguments(audio_path), stage: "Audio preparation")
+          run!(transcription_arguments(audio_path, output_prefix), stage: "Whisper transcription")
+          read_transcript("#{output_prefix}.json")
+        end
+
+        success(payload: transcript)
+      end
+
+      def preparation_arguments(audio_path)
+        ["ffmpeg", "-nostdin", "-i", path, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", "-y", audio_path]
+      end
+
+      def transcription_arguments(audio_path, output_prefix)
+        arguments = [executable, "-m", model_path, "-f", audio_path, "-t", threads.to_s, "--output-json", "-of", output_prefix]
+        arguments.concat(["-l", language]) if language
+        arguments
+      end
+
+      def run!(arguments, stage:)
+        result = Cli::Runner.call(argv: arguments)
+        raise TranscriptionError, "#{stage} failed: #{result.error.message}" if result.failure?
+      end
+
+      def read_transcript(output_path)
+        output = JSON.parse(File.read(output_path))
+        result = ResponseParser.call(output: output, original_path: path)
+        raise TranscriptionError, "Invalid Whisper transcript: #{result.error.message}" if result.failure?
+
+        result.payload
+      rescue Errno::ENOENT
+        raise TranscriptionError, "Whisper did not produce a JSON transcript"
+      rescue JSON::ParserError => e
+        raise TranscriptionError, "Invalid Whisper JSON: #{e.message}"
+      end
+    end
+  end
+end

data/lib/substrate/transcripts/select_time_range.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Transcripts
+    class SelectTimeRange < Command
+      def initialize(content:, range:)
+        @content = content
+        @range = range
+      end
+
+      def call
+        success(payload: Content.new(
+          source_ref: content.source_ref,
+          segments: selected_segments,
+          metadata: content.metadata
+        ))
+      end
+
+      private
+
+      attr_reader :content, :range
+
+      def selected_segments
+        content.segments.filter_map do |segment|
+          next unless overlaps?(segment)
+
+          selected_segment(segment)
+        end
+      end
+
+      def selected_segment(segment)
+        words = selected_words(segment.words)
+
+        Segment.new(
+          text: selected_text(segment, words),
+          start: [segment.start, range.begin].max,
+          end: [segment.end, range.end].min,
+          confidence: segment.confidence,
+          speaker: segment.speaker,
+          words: words
+        )
+      end
+
+      def selected_words(words)
+        words.select { |word| overlaps?(word) }
+      end
+
+      def selected_text(segment, words)
+        return words.map(&:text).join(" ") if words.any?
+
+        segment.text
+      end
+
+      def overlaps?(timed_item)
+        timed_item.start < range.end && timed_item.end > range.begin
+      end
+    end
+  end
+end

data/lib/substrate/url_resolvers/generic.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Substrate
+  module UrlResolvers
+    class Generic < Command
+      def initialize(url:)
+        @url = url
+      end
+
+      def call
+        result = Nous.fetch(url, details: true)
+        page = result.pages.first
+
+        return success(payload: []) unless page
+
+        metadata = build_metadata(page)
+        article = build_article(page, metadata)
+        candidate = build_candidate(article, page, metadata)
+
+        success(payload: [candidate])
+      rescue
+        success(payload: [])
+      end
+
+      private
+
+      attr_reader :url
+
+      def build_metadata(page)
+        Metadata.new(
+          requested_url: page.metadata[:requested_url],
+          final_url: page.url,
+          redirected: page.metadata[:redirected],
+          content_type: page.metadata[:content_type],
+          extractor: page.metadata[:extractor]
+        )
+      end
+
+      def build_article(page, metadata)
+        Sources::Article.new(
+          url: url,
+          text: page.content,
+          title: page.title,
+          metadata: metadata
+        )
+      end
+
+      def build_candidate(article, page, metadata)
+        UrlCandidate.new(
+          id: candidate_id(url, page),
+          kind: :article,
+          source: article,
+          rank: 1,
+          confidence: confidence_for(page),
+          label: "Article",
+          metadata: metadata
+        )
+      end
+
+      def candidate_id(url, page)
+        Digest::MD5.hexdigest("#{url}:#{page.title}")
+      end
+
+      def confidence_for(page)
+        return 0.5 unless page.title && page.content
+
+        (page.title.empty? || page.content.empty?) ? 0.5 : 0.8
+      end
+    end
+  end
+end

data/lib/substrate/url_resolvers/resolver_for.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Substrate
+  module UrlResolvers
+    class ResolverFor < Command
+      class ResolutionError < StandardError; end
+
+      def initialize(url:)
+        @url = url
+      end
+
+      def call
+        resolver = resolver_for(uri)
+        result = resolver.call(url: url)
+        raise ResolutionError, result.error.message unless result.success?
+
+        success(payload: result.payload)
+      end
+
+      private
+
+      attr_reader :url
+
+      def uri
+        @uri ||= URI.parse(url)
+      end
+
+      def resolver_for(_uri)
+        Generic
+      end
+    end
+  end
+end

data/lib/substrate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Substrate
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/substrate.rb

@@ -1,8 +1,21 @@
 # frozen_string_literal: true
 
-require_relative "substrate/version"
+require "zeitwerk"
+require "digest"
+require "nous"
+require "uri"
 
 module Substrate
-  class Error < StandardError; end
-  # Your code goes here...
+  class << self
+    def loader
+      @loader ||= begin
+        loader = Zeitwerk::Loader.for_gem
+        loader.inflector.inflect("pdf" => "Pdf")
+        loader.collapse("#{__dir__}/substrate/primitives")
+        loader
+      end
+    end
+  end
+
+  loader.setup
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: substrate
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Dan Frenette
@@ -9,8 +9,36 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: hexapdf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.1
+- !ruby/object:Gem::Dependency
+  name: marcel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -23,6 +51,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nous
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -42,6 +84,70 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/substrate.rb
+- lib/substrate/classify.rb
+- lib/substrate/cli/availability.rb
+- lib/substrate/cli/error.rb
+- lib/substrate/cli/runner.rb
+- lib/substrate/command.rb
+- lib/substrate/configuration.rb
+- lib/substrate/detection/path_resolver.rb
+- lib/substrate/distill.rb
+- lib/substrate/extract_pdf.rb
+- lib/substrate/handlers/article.rb
+- lib/substrate/handlers/pdf.rb
+- lib/substrate/handlers/raw_text.rb
+- lib/substrate/handlers/text_file.rb
+- lib/substrate/handlers/transcript.rb
+- lib/substrate/mediums/pdf/box.rb
+- lib/substrate/mediums/pdf/chapter.rb
+- lib/substrate/mediums/pdf/content.rb
+- lib/substrate/mediums/pdf/destination.rb
+- lib/substrate/mediums/pdf/destination_resolver.rb
+- lib/substrate/mediums/pdf/document.rb
+- lib/substrate/mediums/pdf/encrypted_pdf.rb
+- lib/substrate/mediums/pdf/entry.rb
+- lib/substrate/mediums/pdf/invalid_pdf.rb
+- lib/substrate/mediums/pdf/invalid_text.rb
+- lib/substrate/mediums/pdf/limit_exceeded.rb
+- lib/substrate/mediums/pdf/limits.rb
+- lib/substrate/mediums/pdf/named_destinations.rb
+- lib/substrate/mediums/pdf/no_selectable_text.rb
+- lib/substrate/mediums/pdf/outline.rb
+- lib/substrate/mediums/pdf/outline_facts.rb
+- lib/substrate/mediums/pdf/outline_parser.rb
+- lib/substrate/mediums/pdf/page.rb
+- lib/substrate/mediums/pdf/page_extractor.rb
+- lib/substrate/mediums/pdf/reader.rb
+- lib/substrate/mediums/pdf/text_extractor.rb
+- lib/substrate/primitives/article_content.rb
+- lib/substrate/primitives/extraction.rb
+- lib/substrate/primitives/metadata.rb
+- lib/substrate/primitives/page_collection.rb
+- lib/substrate/primitives/pdf_file.rb
+- lib/substrate/primitives/raw_text.rb
+- lib/substrate/primitives/source.rb
+- lib/substrate/primitives/text_content.rb
+- lib/substrate/primitives/text_file.rb
+- lib/substrate/primitives/transcripts/content.rb
+- lib/substrate/primitives/transcripts/metadata.rb
+- lib/substrate/primitives/transcripts/segment.rb
+- lib/substrate/primitives/transcripts/word.rb
+- lib/substrate/primitives/url_candidate.rb
+- lib/substrate/source_ref.rb
+- lib/substrate/sources/article.rb
+- lib/substrate/sources/audio.rb
+- lib/substrate/sources/pdf.rb
+- lib/substrate/sources/text.rb
+- lib/substrate/sources/url.rb
+- lib/substrate/transcribers/deepgram.rb
+- lib/substrate/transcribers/deepgram/client.rb
+- lib/substrate/transcribers/deepgram/configuration.rb
+- lib/substrate/transcribers/deepgram/response_parser.rb
+- lib/substrate/transcribers/whisper_cpp.rb
+- lib/substrate/transcribers/whisper_cpp/response_parser.rb
+- lib/substrate/transcripts/select_time_range.rb
+- lib/substrate/url_resolvers/generic.rb
+- lib/substrate/url_resolvers/resolver_for.rb
 - lib/substrate/version.rb
 licenses: []
 metadata: {}