dubbletrack_remote 0.7.1

data/lib/dubbletrack_remote/cli.rb

@@ -0,0 +1,353 @@
+require "thor/rake_compat"
+require "rspec/core/rake_task"
+require "sinatra/activerecord/rake"
+require "pry"
+require "fileutils"
+
+module DubbletrackRemote
+  class CLI < Thor
+    include Thor::Actions
+    include Thor::RakeCompat
+
+    class_option :debug, type: :boolean, default: false
+    class_option :dir, type: :string, default: "/opt/dubbletrack-remote/"
+
+    desc "watch", "watch playlists folder and post on changes"
+    def watch
+      ensure_setup
+
+      Reader.watch(watch_path, pattern: watch_pattern) do |file|
+        puts "detected change in #{file}"
+        Reader.new(file, {cuts_path: local_cuts_file}).ingest
+      end
+
+      Thread.new do
+        loop do
+          Item.next(20).each do |item|
+            client.send([item])
+          end
+          update # this feels redundant with the watch above
+          status
+        rescue => e
+          puts "errored, but will try to move on"
+          puts e
+        ensure
+          sleep 10
+        end
+      end
+
+      backfill
+
+      sleep
+    end
+    default_task :watch
+
+    desc "status", "get status"
+    def status
+      ensure_setup
+      puts "total remaining: #{Item.remaining.count}"
+      puts "total transmitted: #{Item.successful.count}"
+      puts "recent items needing transmission: #{Item.recent_remaining.count}"
+    end
+
+    desc "config", "open config file"
+    def config
+      system("open #{settings_path}")
+    end
+
+    desc "setup", "setup dubbletrack remote for the first time"
+    def setup
+      create_settings_file
+      ensure_setup
+    end
+
+    desc "update", "post latest tracks from current playlist"
+    def update
+      ensure_setup
+
+      if todays_playlist_path
+        [todays_playlist_path].flatten.each { |f| Reader.new(f, {cuts_path: local_cuts_file}).ingest }
+      else
+        puts "no file found matching pattern #{playlist_pattern}"
+      end
+    end
+
+    desc "post FILE_PATH or DATES", "post latest track from playlist"
+    def post(*args)
+      ensure_setup
+      paths = args_to_paths(args)
+
+      if paths.any?
+        items = paths.collect do |path|
+          puts "reading #{path}"
+          items = Reader.new(path, {cuts_path: local_cuts_file}).items
+        end.flatten.sort_by(&:played_at)
+        remaining = Item.where(id: items.pluck(:id)).remaining
+
+        if remaining.count > 0
+          puts "sending #{remaining.size} from playlist #{file_path}"
+          Item.where(id: remaining.pluck(:id)).remaining.find_each do |item|
+            client.send(item)
+          end
+        end
+      else
+        puts "could not find any files with args: #{args}"
+      end
+    end
+
+    # desc "next SIZE", "check the next up to be sent"
+    # def next(size = 20)
+    #   ensure_setup
+
+    #   Item.next(size).each do |item|
+    #     puts item.pretty_print
+    #   end
+    # end
+
+    desc "read FILE_PATH or DATES", "read tracks from playlist"
+    option :raw, type: :boolean, description: "Show raw fields?", default: false
+    def read(*args)
+      ensure_setup
+      paths = args_to_paths(args)
+      if paths.any?
+        items = paths.collect do |path|
+          puts "reading #{path}"
+          items = Reader.new(path, {cuts_path: local_cuts_file}).items
+        end.flatten.sort_by(&:played_at)
+
+        items.each do |e|
+          puts e.pretty_print
+          puts e.raw if options[:raw]
+        end
+      else
+        puts "could not find any files with args: #{args}"
+      end
+    end
+
+    desc "import FILE_PATH or DATES", "import tracks from playlist"
+    def import(*args)
+      ensure_setup
+      paths = args_to_paths(args)
+      if paths.any?
+        paths.each do |path|
+          puts "importing #{path}"
+          items = Reader.new(path, {cuts_path: local_cuts_file}).ingest
+          items.each { |e| puts e.pretty_print }
+        end
+      else
+        puts "could not find any files with args:"
+      end
+    end
+
+    desc "backfill", "read previous playlists and post changes"
+    def backfill(from_date = nil)
+      ensure_setup
+
+      from_date = from_date ? Date.parse(from_date) : Date.today - 7.days
+
+      paths = []
+      while from_date < Date.today
+        paths << playlist_path_from_date(from_date)
+        from_date += 1.day
+      end
+
+      paths.flatten!
+
+      if paths.any?
+        paths.each do |path|
+          puts "importing #{path}"
+          items = Reader.new(path, {cuts_path: local_cuts_file}).ingest
+          items.each { |e| puts e.pretty_print }
+        end
+      else
+        puts "could not find any files with args:"
+      end
+    end
+
+    # desc "list", "list all available files based on settings"
+    # def list
+    #   ensure_setup
+    #   playlist_pattern, _ = settings[:automation][:playlist_path].split("%")
+    #   playlist_pattern = "#{playlist_pattern}*"
+    #   files = Dir.glob(playlist_pattern)
+    #   files.map { |f| File.expand_path(f) }
+    # end
+
+    desc "console", ""
+    def console
+      ensure_setup
+
+      # turns on logging of SQL queries while in the task
+      ActiveRecord::Base.logger = Logger.new(STDOUT)
+      # starts a Ruby REPL session
+      Pry.start
+    end
+
+    desc "version", "prints current dubbletrack_remote version"
+    def version
+      ensure_setup
+
+      puts "version: #{DubbletrackRemote::VERSION}"
+    end
+
+    # no_commands do
+    #   def options
+    #     super
+    #     # original_options = super
+    #     # defaults =  if original_options[:dir] # the settings path was passed in
+    #     #               config = File.expand_path(File.join(original_options[:dir], settings_file_name))
+    #     #               if File.exist?(config)
+    #     #                 ::YAML.load_file(config) || {}
+    #     #               else
+    #     #                 {}
+    #     #               end
+    #     #             end
+    #     # return Thor::CoreExt::HashWithIndifferentAccess.new(defaults.merge(original_options))
+    #   end
+    # end
+
+    protected
+
+    def update_current_cuts_file?
+      return true if find_local_cuts_file.blank?
+
+      latest_time = Time.at(File.basename(find_local_cuts_file, File.extname(find_local_cuts_file)).split("_").last.to_i)
+      latest_time < 6.hours.ago
+    end
+
+    def delete_old_cuts
+      deletable = Dir.glob(File.join(options[:dir], "CUTS_*.DBF")).sort.reverse.slice(1...-1) || []
+      deletable.each { |to_delete| FileUtils.rm(to_delete) }
+    end
+
+    def find_local_cuts_file
+      cuts = Dir.glob(File.join(options[:dir], "CUTS_*.DBF")).sort.reverse
+      cuts.last
+    end
+
+    def local_cuts_file
+      if remote_cuts_path = Settings.automation.cuts_path
+        if update_current_cuts_file?
+          current_cut_path = File.join(options[:dir], "CUTS_#{Time.now.to_i}.DBF")
+          delete_old_cuts
+          FileUtils.cp(remote_cuts_path, current_cut_path)
+
+          return current_cut_path
+        end
+
+        find_local_cuts_file
+      end
+    end
+
+    def args_to_paths(args)
+      paths = []
+      args.each do |arg|
+        if File.exist?(arg)
+          paths << arg
+        elsif arg.scan(/\d\d\d\d-\d\d-\d\d/)
+          paths << playlist_path_from_date(Time.zone.parse(arg))
+        end
+      end
+
+      paths = paths.flatten.compact
+    end
+
+    def client
+      @client ||= Client.new(
+        url: settings[:api_url],
+        key: settings[:api_key],
+        secret: settings[:api_secret]
+      )
+    end
+
+    def watch_path
+      path = File.expand_path(settings.dig(:automation, :playlist_path))
+      File.dirname(path)
+    end
+
+    def watch_pattern
+      path = File.expand_path(settings.dig(:automation, :playlist_path))
+      Regexp.new(File.basename(path).gsub(/%\w/, ".+"), "i")
+    end
+
+    def watch_extension
+      path = File.expand_path(settings.dig(:automation, :playlist_path))
+      Regexp.new("#{File.extname(path).downcase}$", "i")
+    end
+
+    def todays_playlist_path
+      playlist_path_from_date(Time.now)
+    end
+
+    def playlist_path_from_date(date = Time.now)
+      playlist_pattern = date.strftime(settings[:automation][:playlist_path])
+      files = Dir.glob(playlist_pattern)
+      files.map { |f| File.expand_path(f) }
+    end
+
+    def settings
+      Settings
+    end
+
+    def settings_path
+      File.expand_path(File.join(options[:dir], settings_file_name))
+    end
+
+    def settings_file_name
+      ".dubbletrack-remote-settings"
+    end
+
+    def ensure_setup
+      connect_db
+      if !File.exist?(settings_path)
+        if yes? "dubbletrack_remote is not setup at #{options[:dir]}. Setup? (y/n)"
+          create_settings_file
+        end
+      end
+      Config.load_and_set_settings(settings_path)
+      Time.zone = Settings.automation.time_zone
+      begin
+        Rake::Task["db:migrate"].invoke
+      rescue
+      end
+    end
+
+    def create_settings_file
+      content = ""
+      create_file(settings_path) do
+        station_id = ask "What's your dubbletrack station ID? (e.g. koop, kvrx)"
+        api_key = ask "What's your dubbletrack API Key?"
+        api_secret = ask "What's your dubbletrack API Secret?", echo: false
+        automation_type = ask "What kind of automation system is this?", default: "enco"
+        dump_delay = ask "How much of a delay (in seconds) is there before broadcast? (i.e. do you have a dump button)", default: 0
+        time_zone = ask "What timezone are you in?", default: "America/Chicago"
+
+        if automation_type == "enco"
+          format = "dbf"
+          cuts_path = ask "Where is the cuts database? (CUTS.DBF)", path: true
+          playlists_path = ask "Where should I look for playlists? (This can include regex and strftime components, i.e. /Enco/Dad/STUDIO[1-2]%m%d%y.DBF)", path: true
+        end
+        <<~YAMS.strip
+          api_url: https://api.dubbletrack.com/api/v1/stations/#{station_id}/automation
+          api_key: #{api_key}
+          api_secret: #{api_secret}
+          automation:
+            time_zone: #{time_zone}
+            type: #{automation_type}
+            playlist_path: #{playlists_path}
+            cuts_path: #{cuts_path}
+            delay_seconds: #{dump_delay}
+        YAMS
+      end
+    end
+
+    def connect_db
+      unless ENV["DUBBLETRACK_REMOTE_ENV"] == "test"
+        Sinatra::Application.set :database, {
+          adapter: "sqlite3",
+          database: File.expand_path(File.join(options[:dir], "dubbletrack-remote.sqlite3"))
+        }
+      end
+    end
+  end
+end

data/lib/dubbletrack_remote/client.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module DubbletrackRemote
+  class Client
+    def initialize(url:, key:, secret:)
+      @client = Faraday.new(url: url) do |faraday|
+        faraday.request :retry,
+          max: 2,
+          interval: 0.05,
+          interval_randomness: 0.5,
+          backoff_factor: 2
+
+        faraday.adapter Faraday.default_adapter # make requests with Net::HTTP
+      end
+
+      @url = url # || 'https://dubbletrack.com/api/v1/stations/koop/shows/koop-automation/tracks'
+      @key = key # || 'dt_xxxxxxxx'
+      @secret = secret # || 'dts_xxxxxxxxxxxx'
+    end
+
+    def send(items)
+      items = [items].flatten
+
+      payload = JSON.generate(
+        data: items.collect { |item| item_json(item) }
+      )
+
+      headers = {
+        :"Content-Type" => "application/json",
+        :"X-API-KEY" => @key,
+        :"X-API-SECRET" => @secret,
+        "User-Agent" => "Dubbletrack Remote"
+      }
+
+      items.each do |i|
+        i.last_attempt_at = Time.now.iso8601
+      end
+
+      begin
+        response = @client.post(@url, payload, headers)
+        if response.success?
+          data = JSON.parse(response.body).dig("data")
+
+          begin
+            items.each do |item|
+              item.success = true
+              item.last_error_text = nil
+              item.last_error_code = nil
+
+              found_response = if data.size > 1
+                data.find do |d|
+                  d["attributes"]["title"] == item.title &&
+                    d["attributes"]["artist_name"] == item.artist_name
+                end
+              else
+                data.first
+              end
+              item.remote_id = found_response["id"] if found_response
+              item.save
+              puts item.pretty_print
+            end
+          rescue => e
+            puts e
+          end
+        else
+          items.each do |item|
+            item.success = false
+            error = DubbletrackRemote::Error.new(response)
+            raise DubbletrackRemote::RateLimitError if error.text == "Rate Limit Exceeded"
+            item.last_error_text = error.text
+            item.last_error_code = error.code
+            item.save
+            puts item.pretty_print
+          end
+        end
+      rescue Faraday::ConnectionFailed => exception
+        items.each do |item|
+          item.last_error_text = "connection failed"
+          item.last_error_code = 404
+          item.save
+        end
+      end
+    end
+
+    private
+
+    def key_map(key)
+      map = {
+        automation_group: :remote_automation_group,
+        automation_id: :remote_automation_id,
+        automation_system: :remote_automation_system,
+        source: :remote_automation_source,
+        guid: :remote_automation_guid
+      }
+
+      map[key] || key
+    end
+
+    def item_json(item)
+      attributes = [
+        :title,
+        :artist_name,
+        :release_name,
+        :label_name,
+        :played_at,
+        :duration,
+        :genre,
+        :automation_group,
+        :automation_id,
+        :automation_system,
+        :source,
+        :guid
+      ].each_with_object({}) do |key, hash|
+        value = begin
+          item.send(key)
+        rescue
+          nil
+        end
+        if value && key_map(key).is_a?(Symbol)
+          hash[key_map(key)] = value
+        elsif key_map(key).is_a?(String)
+          hash[key] = key_map(key)
+        end
+
+        if key == :played_at && item.respond_to?(:played_at_with_delay)
+          hash[:played_at] = item.send(:played_at_with_delay)
+        end
+      end
+
+      {
+        type: item.item_type,
+        attributes: attributes
+      }
+    end
+  end
+end

data/lib/dubbletrack_remote/errors.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module DubbletrackRemote
+  class RateLimitError
+  end
+
+  class Error
+    attr_accessor :time, :text, :code
+
+    def initialize(response)
+      @time = Time.now.iso8601
+      @code = response.status
+      @text = error_text(response)
+    end
+
+    def error_text(response)
+      error_text = response.body
+      begin
+        json = JSON.parse(response.body)
+        errors = json["errors"] || []
+        error_text = errors.flatten.collect { |error| error["detail"] }.join(",")
+      rescue JSON::ParserError
+        # keep the error text as response.body
+      end
+      error_text
+    end
+
+    def to_json
+      {
+        text: @text,
+        code: @code,
+        time: @time
+      }
+    end
+  end
+end

data/lib/dubbletrack_remote/item.rb

@@ -0,0 +1,79 @@
+# An item to be sent to dubbletrack. Either a track or a traffic update, can be created from a hash or a db record
+# module DubbletrackRemote
+module DubbletrackRemote
+  class Item < ActiveRecord::Base
+    TRACK = "track"
+    TRAFFIC = "traffic"
+
+    # TODO: fix these scopes so they're not so horrendous, is there a way to make sqlite play nicer with booleans?
+
+    scope :successful, -> { where(["success = ? OR success = ?", 1, true]) }
+    scope :known, -> { where("item_type IN (?)", [TRACK, TRAFFIC]) }
+    scope :remaining, -> { known.where(["(success IS NULL OR success = ?) AND (last_error_text IS NULL OR last_error_text != ?)", false, "Played at has already been taken"]) }
+    scope :tracks, -> { where(["item_type = ?", TRACK]) }
+    scope :traffic, -> { where(["item_type = ?", TRAFFIC]) }
+    scope :recent_remaining, -> { remaining.where(["played_at > ?", 1.day.ago]) }
+    scope :next, ->(count = 10) { remaining.order("played_at desc").limit(count) }
+    scope :on_date, ->(time) { where(["played_at >= ? AND played_at <= ?", time.beginning_of_day, time.end_of_day]) }
+
+    validates :played_at, uniqueness: true
+    # validates :intended_played_at, uniqueness: true, allow_blank: true
+
+    before_validation :set_item_type
+    before_save :set_item_type
+
+    attr_accessor :raw
+
+    def played_at_with_delay
+      played_at + (Settings&.automation&.delay_seconds || 0).seconds
+    rescue
+      played_at
+    end
+
+    def set_item_type
+      self.item_type = detect_item_type
+    end
+
+    def detect_item_type
+      return TRACK if track?
+      return TRAFFIC if traffic?
+    end
+
+    def key
+      "#{played_at.utc.iso8601}-#{title}:#{artist_name}"
+    end
+
+    def track?
+      title.present? && played_at.present? && artist_name.present? && (label_name.present? || release_name.present?) # label name and release name could technically be left out, even though they shouldn't
+    end
+
+    def traffic?
+      title.present? && played_at.present? && !track?
+    end
+
+    def pretty_print
+      state = if success
+        "[✓] "
+      elsif last_error_code
+        "[×] "
+      else
+        "[ ] "
+      end
+      duration_s = duration ? " [#{duration}s]" : ""
+      if track?
+        line = "TRACK: #{state}#{id}:#{played_at.utc.iso8601}".ljust(30) + "#{title} - #{artist_name}" "\r\n" +
+          " ".ljust(32) + "#{release_name} // #{label_name} \r\n" +
+          " ".ljust(32) + "#{genre}, #{duration_s}, #{automation_group}, #{automation_id}" + "\r\n" +
+          (last_error_code ? "".ljust(30) + "[#{last_error_code}] #{last_error_text}" : "") + "\r\n"
+      elsif traffic?
+        line = "TRAFFIC: #{state}#{id}:#{played_at.utc.iso8601}".ljust(30) + title.to_s + "\r\n" +
+          "#{genre}, #{duration_s}, #{automation_group}, #{automation_id}" + "\r\n" +
+          (last_error_code ? "".ljust(30) + "[#{last_error_code}] #{last_error_text}" : "") + "\r\n"
+      else
+        line = "UNKNOWN: #{state}#{id}:#{played_at.utc.iso8601}".ljust(30) + title.to_s + "\r\n" +
+          "#{genre}, #{duration_s}, #{automation_group}, #{automation_id}" + "\r\n" +
+          (last_error_code ? "".ljust(30) + "[#{last_error_code}] #{last_error_text}" : "") + "\r\n"
+      end
+    end
+  end
+end

data/lib/dubbletrack_remote/reader/base.rb

@@ -0,0 +1,57 @@
+# Reads in a DBF file and creates items out of them, annotating with information from CUTS
+module DubbletrackRemote
+  module Reader
+    def self.new(file_path, options = {})
+      if File.extname(file_path) == ".DBF"
+        DubbletrackRemote::Reader::DBF.new(file_path, options)
+      elsif File.extname(file_path) == ".TSV"
+        DubbletrackRemote::Reader::TSV.new(file_path, options)
+      end
+    end
+
+    def self.watch(playlist_path, options = {}, &block)
+      pattern = options[:pattern] || Regexp.new(".+")
+      listener = Listen.to(playlist_path, force_polling: true, wait_for_delay: 4) do |modified, added|
+        [modified, added].flatten.compact.uniq.each do |file|
+          if pattern.match(file)
+            block.call(file)
+          end
+        end
+      rescue => e
+        puts "errored while listening for changes, will continue listening"
+        puts e
+      end
+
+      listener.start
+    end
+
+    class Base
+      attr_reader :use_database
+
+      def items
+        @items ||= read_items.sort_by(&:played_at)
+      end
+
+      def tracks
+        items.select { |i| i.item_type == DubbletrackRemote::Item::TRACK }
+      end
+
+      def traffic
+        items.select { |i| i.item_type == DubbletrackRemote::Item::TRAFFIC }
+      end
+
+      def ingest
+        items.select { |s| s.changed? || s.new_record? }.each do |item|
+          puts item.to_json
+          item.save
+        end
+
+        items
+      end
+
+      def read_items
+        raise "must implement in subclass"
+      end
+    end
+  end
+end