fcc 1.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1a2e65473c73d939b88beed853dfe8cb2ab4bb67684c522996a4b1eb0f8d7f8
-  data.tar.gz: d86f808b6da433d743ac6e2546307cf4bdfeb42ec521796cb78a0ecfffb7da63
+  metadata.gz: 0a40f16f549528352153a0d8fb4c1bed24c966b9c3647ae7aee71f46b419920f
+  data.tar.gz: db47ff830e6c821ffce6fa3c07bd1bb75f34dfb60ca1ffd85566cf6cb95ce7a4
 SHA512:
-  metadata.gz: e6a3bea8e717984ddd25fed17731e3ba06709c70dcb49340a274d148fcdd281e1463baf6483879d9a2427961589f2e0d7af908d8ce4ab6508776c8ba1d80c5e3
-  data.tar.gz: 6da0ba2408b94f8488c45900938d3950e580a5cb71493b942a78696b462f0bc4dee61ea26b5bf4cbd1c54a155ad0b0c5670cde37e90c99ede9581b196cd75586
+  metadata.gz: 14d033f2b67c0d3a1e8528fd2d4d787c3bf1be01e2f438feb0df731a0d39e5ab6695e721e4b292e2561efb6374591080186b20eff77b82c26f3fcb8df5556ddd
+  data.tar.gz: 0a9da9bf1d013f096a6c9eb4cc798b8cf98546024702e7c774fe0303d82800a6f1235a0303438b6162585a6673bcd2d6cb7d7e0aadf0b4406bcfa87dc8273f2b

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+fcc changelog
+
+## [1.3.0](https://github.com/jkeen/fcc/compare/v1.3.0...v1.2.1) (2022-07-10)
+
+- Combine data from multiple sources to get a full picture of the station data (transition api, public api, LMS csv files)
+- Now includes related transmitters, repeaters, low power stations, etc

data/Gemfile

@@ -5,19 +5,7 @@ git_source(:github) { |repo_name| 'https://github.com/jkeen/fcc' }
 # Specify your gem's dependencies in popularity.gemspec
 gemspec
 
-#
-# source "http://rubygems.org"
-# # Add dependencies required to use your gem here.
-# # Example:
-# #   gem "activesupport", ">= 2.3.5"
-#
-# # Add dependencies to develop your gem here.
-# # Include everything needed to run rake, tests, features, etc.
-#
-# gem "nokogiri"
-#
 # group :development do
-#   gem "shoulda", ">= 0"
-#   gem "bundler"
-#   gem "jeweler"
+#   gem "awesome_print", require: true
+#   gem "byebug", require: true
 # end

data/Gemfile.lock

@@ -1,32 +1,34 @@
 PATH
   remote: .
   specs:
-    fcc (1.1)
+    fcc (1.3.0)
       activesupport (>= 6.1)
       httparty (~> 0.18)
+      lightly (~> 0.3.3)
+      logger
+      rubyzip
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.0)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
-    awesome_print (1.8.0)
-    byebug (11.1.3)
-    concurrent-ruby (1.1.7)
-    diff-lcs (1.4.4)
-    httparty (0.18.1)
+    concurrent-ruby (1.1.10)
+    diff-lcs (1.5.0)
+    httparty (0.20.0)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    i18n (1.8.6)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    mime-types (3.3.1)
+    lightly (0.3.3)
+    logger (1.5.0)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
-    minitest (5.14.2)
+    mime-types-data (3.2022.0105)
+    minitest (5.16.2)
     multi_xml (0.6.0)
     rake (12.3.3)
     rspec (3.9.0)
@@ -42,17 +44,15 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.4)
+    rubyzip (2.3.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  awesome_print
   bundler (~> 2.1)
-  byebug
   fcc!
   rake (~> 12.3.3)
   rspec (~> 3.9.0)

data/README.md

@@ -2,6 +2,9 @@
 
 Query the FCC database for fm, am, and tv information.
 
+
+This gem uses the new FCC apis and the old FCC apis and combines records to be the most complete version possible. The old (transition.fcc.gov) API provides way more data than the new one does.
+
 ## Install
 
 ```
@@ -12,35 +15,78 @@ gem install fcc
 
 * Ruby 2.0.0 or higher
 
-## Examples
-
 #### Get station information by call sign
 
 ```ruby
 station = FCC::Station.find(:fm, "KOOP")
 
-#Basic attributes, available quickly because the FCC actually caches these in a CDN: 
-station.id #=> 65320
-station.status #=> LICENSED
-station.rf_channel #=> 219
-station.license_expiration_date #=> "08/01/2021"
-station.facility_type #=> ED
-station.frequency #=> 91.7 
-station.contact #=> <struct FCC::Station::Contact>
-station.owner #=> <struct FCC::Station::Contact>
-station.community #=> <struct FCC::Station::Community city="HORNSBY", state="TX">
-
-#Extended attributes, takes several seconds to load because the FCC is running this endpoint on a 1960s era mainframe operated by trained hamsters
-station.station_class #=> A
-station.signal_strength #=> 3.0 kW
-station.antenna_type #=> ND
-station.effective_radiated_power #=> 3.0 kW
-station.haat_horizontal #=> 26.0
-station.haat_vertical #=> 26.0
-station.latitude #=> "30.266861111111112"
-station.longitude #=> "-97.67444444444445"
-station.file_number #=> BLED-19950103KA
-```
+if station.exists? && station.licensed?
+  #Basic attributes, available quickly because the FCC actually caches these in a CDN: 
+  station.id #=> 65320
+  station.status #=> LICENSED
+  station.rf_channel #=> 219
+  station.license_expiration_date #=> "08/01/2021"
+  station.facility_type #=> ED
+  station.frequency #=> 91.7 
+  station.contact #=> <struct FCC::Station::Contact>
+  station.owner #=> <struct FCC::Station::Contact>
+  station.community #=> <struct FCC::Station::Community city="HORNSBY", state="TX">
+
+  # Extended attributes, takes several seconds to load initially because the FCC is running this endpoint on a 1960s era mainframe operated by trained hamsters. 
+  station.station_class #=> A
+  station.signal_strength #=> 3.0 kW
+  station.antenna_type #=> ND
+  station.effective_radiated_power #=> 3.0 kW
+  station.haat_horizontal #=> 26.0
+  station.haat_vertical #=> 26.0
+  station.latitude #=> "30.266861111111112"
+  station.longitude #=> "-97.67444444444445"
+  station.file_number #=> BLED-19950103KA
+
+
+  station.to_json #=> 
+                [{:band=>"FM",
+                :signal_strength=>"3.0 kW",
+                :latitude=>"30.266861111111112",
+                :longitude=>"-97.67444444444445",
+                :station_class=>"A",
+                :file_number=>"BLED-19950103KA",
+                :effective_radiated_power=>"3.0 kW",
+                :haat_horizontal=>"26.0",
+                :haat_vertical=>"26.0",
+                :antenna_type=>"ND",
+                :operating_hours=>nil,
+                :licensed_to=>"TEXAS EDUCATIONAL BROADCASTING CO-OPERATIVE, INC.",
+                :city=>"HORNSBY",
+                :state=>"TX",
+                :country=>"US",
+                :id=>"65320",
+                :call_sign=>"KOOP",
+                :status=>"LICENSED",
+                :rf_channel=>"219",
+                :license_expiration_date=>"08/01/2029",
+                :facility_type=>"ED",
+                :frequency=>"91.7",
+                :contact=>
+                 {:name=>"Federico Pacheco",
+                  :title=>"",
+                  :address=>"3823 Airport Blvd. Suite B",
+                  :address2=>"",
+                  :city=>"Austin",
+                  :state=>"TX",
+                  :zip_code=>"78722",
+                  :phone=>"(512)472-1369",
+                  :fax=>"",
+                  :email=>"federico.pacheco@koop.org",
+                  :website=>"koop.org"},
+                :owner=>{:name=>"TEXAS EDUCATIONAL BROADCASTING CO-OPERATIVE, INC.", :city=>"AUSTIN", :state=>"TX", :phone=>"(512)472-1369"},
+                :community=>{:city=>"HORNSBY", :state=>"TX"}}]
+  ```
+end
+
+
+### Caching
+Extended attributes take several seconds to load from transition.fcc.gov. In order to work around this, we query the entire dataset and then cache the result locally for 3 days (using the lightly gem). To use your own cache, set `FCC.cache=` to your cache class (Rails.cache, maybe?) which should have a #fetch method that should take a key and a block like `cache.fetch(key) { // yield for expensive fetch }}`.  
 
 ### Get all station call signs on a particular service (:fm, :am, :tv)
 

data/VERSION

@@ -1 +1 @@
-0.1.0
+1.2.1

data/fcc.gemspec

@@ -6,7 +6,6 @@ Gem::Specification.new do |spec|
   spec.name = %q{fcc}
   spec.version = FCC::VERSION
   spec.authors = ["Jeff Keen"]
-  spec.date = %q{2011-01-30}
   spec.description = %q{}
   spec.email = %q{jeff@keen.me}
   spec.homepage      = "http://github.com/jkeen/fcc"
@@ -20,9 +19,10 @@ Gem::Specification.new do |spec|
   spec.summary = %q{Searches the FCC's FM, AM, and TV databases}
   spec.add_dependency "activesupport", ">= 6.1"
   spec.add_dependency "httparty", "~> 0.18"
+  spec.add_dependency "lightly", "~> 0.3.3"
+  spec.add_dependency "rubyzip"
+  spec.add_dependency "logger"
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 12.3.3"
   spec.add_development_dependency 'rspec', '~> 3.9.0'
-  spec.add_development_dependency "byebug", ">= 0"
-  spec.add_development_dependency "awesome_print", ">= 0"
 end

data/lib/fcc/station/cache.rb

@@ -1,28 +1,34 @@
 require 'httparty'
-require 'byebug'
-require_relative './extended_info/parser'
+require_relative './parsers/extended_info'
+require 'lightly'
+require 'logger'
 
 module FCC
   module Station
     class Cache
       attr_reader :store
 
-      def initialize(service)
-        @service = service
-        @store = Station::ExtendedInfo.new(@service)
+      def initialize
+        @lightly = Lightly.new(life: '3d', hash: true).tap do |cache|
+          cache.prune
+        end
       end
 
-      def find(fcc_id)
-        results.detect { |r| r[:fcc_id].to_s == fcc_id.to_s }
+      def flush
+        @lightly.flush
       end
 
-      def results
-        #TODO: add redis caching tie-in because this query is molasses
-        @store.all_results.parsed_response
-      end
-
-      def inspect
-        "<Cache @service=#{service}>"
+      def fetch(key)
+        FCC.log("Retreiving #{key} from cache")
+        @lightly.get(key.to_s) do
+          FCC.log "Loading up the cache with results for key: #{key}. This might take a minute…"
+          value = yield
+          if value && value.present?
+            value
+          else
+            nil
+          end
+        end
       end
     end
   end

data/lib/fcc/station/extended_info.rb

@@ -1,26 +1,27 @@
 require 'httparty'
-require 'byebug'
-require_relative './extended_info/parser'
+require_relative './parsers/extended_info'
 
 module FCC
   module Station
     class ExtendedInfo
       include HTTParty
       attr_accessor :results, :service
-      base_uri 'http://transition.fcc.gov/fcc-bin/'
+
+      base_uri 'https://transition.fcc.gov/fcc-bin/'
 
       def initialize(service)
         @service = service
         @options = {
           follow_redirects: true
         }
+        
         @query = {
           # state: nil,
           # call: nil,
           # city: nil,
           # arn: nil,
-          serv: service.to_s.downcase,
-          vac: 3, # licensed records only
+          # serv: service.to_s.downcase, # Only return primary main records, no backup transmitters, etc… for now
+          status: 3, # licensed records only
           # freq: @service.to_sym == :fm ? '87.1' : '530',
           # fre2: @service.to_sym == :fm ? '107.9' : '1700',
           # facid: nil,
@@ -42,33 +43,39 @@ module FCC
       end
 
       def all_results
-        @all_results ||= begin
-          puts "charging cache for #{service} extended info"
-          response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query))
-          puts response.request.uri.to_s.gsub('&list=4', '&list=0')
-          response
+
+        begin
+          cache_key = "#{self.class.instance_variable_get('@default_options')[:base_uri]}/#{@service.to_s.downcase}q"
+          FCC.cache.fetch cache_key do
+            response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query))
+            FCC.log(response.request.uri.to_s.gsub('&list=4', '&list=0'))
+            response.parsed_response
+          end
+        rescue StandardError => e
+          FCC.error(e.message)
+          FCC.error(e.backtrace)
         end
       end
 
       def find(id_or_call_sign)
         if id_or_call_sign =~ /^\d+$/
           id = id_or_call_sign
+          all_results.filter { |r| r[:fcc_id].to_s == id.to_s } || find_directly({ facid: id_or_call_sign })
         else
-          id = FCC::Station.index(@service).call_sign_to_id(id_or_call_sign)
-        end
-
-        begin
-          FCC::Station.extended_info_cache(@service).find(id)
-        rescue Exception => e
-          response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query.merge(facid: id)))
-          puts response.request.uri.to_s.gsub('&list=4', '&list=0')
-          result = response.first
-          result['source_url'] = response.request.uri.to_s.gsub('&list=4', '&list=0')
-          result
+          all_results.filter { |r| 
+            r[:call_sign].to_s == id_or_call_sign.to_s || r[:call_sign].to_s.upcase =~ Regexp.new("^#{id_or_call_sign.upcase}[-—–][A-Z0-9]+$")
+          } || find_directly({ call: id_or_call_sign })
         end
       end
 
-      parser FCC::Station::ExtendedInfoParser
+      def find_directly(options)
+
+        response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query.merge(options)))
+        FCC.log response.request.uri.to_s.gsub('&list=4', '&list=0')
+        response.parsed_response
+      end
+      
+      parser FCC::Station::Parsers::ExtendedInfo
     end
   end
 end

data/lib/fcc/station/index.rb

@@ -1,5 +1,5 @@
 require 'httparty'
-require 'byebug'
+
 module FCC
   module Station
     class Index

data/lib/fcc/station/info.rb

@@ -1,5 +1,4 @@
 require 'httparty'
-require 'byebug'
 
 module FCC
   module Station
@@ -21,7 +20,15 @@ module FCC
              end
 
         response = self.class.get("/api/service/#{service.to_s.downcase}/facility/id/#{id}.json")
-        response['results']['facility']
+
+        begin
+          body = response['results']['facility']
+          body['band'] = service.to_s.upcase.to_s
+
+          body
+        rescue StandardError => e
+          return nil
+        end
       end
     end
   end

data/lib/fcc/station/lms_data.rb

@@ -0,0 +1,139 @@
+require 'httparty'
+require 'open-uri'
+require 'net/http'
+require 'uri'
+require 'date'
+require_relative './parsers/lms_data'
+
+module FCC
+  module Station
+    class LmsData
+      BASE_URI = 'https://enterpriseefiling.fcc.gov/dataentry/api/download/dbfile'
+      # include HTTParty
+
+      def find_all_related(call_sign: )
+        stations = find_related_stations(call_sign: call_sign)
+        translators = find_translators_for(call_sign: stations.keys)
+        stations.merge(translators)
+      end
+
+      def find_translators_for(call_sign: )
+        call_signs = [call_sign].flatten
+
+        records = common_station_data.entries.select do |entry|
+          call_signs.any? { |call_sign| call_signs_match?(call_sign, entry['callsign']) }
+        end 
+
+        facility_ids = records.map { |r| r['facility_id'] }.uniq.compact
+
+        matched_facilities = facilities.entries.select do |facility|
+          facility_ids.include?(facility['primary_station']) #{}|| facility_ids.include?(facility['facility_id'])
+        end
+
+        {}.tap do |hash|
+          matched_facilities.each do |record|
+            hash[record['callsign']] = record['facility_id']
+          end
+        end
+      end
+
+      def find_related_stations(call_sign: )
+        call_signs = [call_sign].flatten
+
+        records = common_station_data.entries.select do |entry|
+          call_signs.any? { call_signs_match?(call_sign, entry['callsign']) }
+        end
+
+        correlated_app_ids = records.map { |m| m['eeo_application_id'] }
+        matches = common_station_data.entries.select do |entry|
+          correlated_app_ids.include?(entry['eeo_application_id'])
+        end
+
+        {}.tap do |hash|
+          matches.each do |record|
+            hash[record['callsign']] = record['facility_id']
+          end
+        end
+      end
+
+      def facilities
+        @facilities ||= read(:facility)
+      end
+
+      def common_station_data
+        @common_station_data ||= read(:common_station)
+      end
+
+      def find_facilities(facility_ids:, call_signs: [])
+        matched_facilities = facilities.entries.select do |facility|
+          facility_ids.include?(facility['primary_station']) || facility_ids.include?(facility['facility_id']) || call_signs.include?(facility['callsign'])
+        end
+
+        {}.tap do |hash|
+          matched_facilities.each do |record|
+            hash[record['callsign']] = record['facility_id']
+          end
+        end
+      end
+
+      def find_call_signs(call_signs:)
+        common_station_data.entries.select do |entry|
+          call_signs.any? do |call_sign|
+            call_signs_match?(call_sign, entry['callsign'])
+          end
+        end
+      end
+
+      def read(file_name)
+        key = "#{lms_date}-#{file_name}"
+        remote_url = URI("#{BASE_URI}/#{lms_date}/#{file_name}.zip")
+        FCC.log remote_url
+        contents = FCC.cache.fetch key do
+          begin
+            temp_file = http_download_uri(remote_url)
+            break if temp_file.empty?
+
+            contents = ""
+            Zip::File.open_buffer(temp_file) do |zf| 
+              contents = zf.read(zf.entries.first)
+              break
+            end
+
+            value = contents
+          rescue Exception => e
+            FCC.error(e.message)
+            value = nil
+          ensure
+            value
+          end
+        end
+
+        if contents
+          CSV.parse(contents, col_sep: '|', headers: true)
+        end
+      end
+
+      protected
+
+      def call_signs_match?(ours, theirs)
+        theirs.to_s.upcase.to_s == ours.to_s.upcase.to_s || theirs.to_s.upcase =~ Regexp.new("^#{ours.to_s.upcase}[-—–][A-Z0-9]+$")
+      end
+
+      def http_download_uri(uri)
+        FCC.log 'Downloading ' + uri.to_s
+        begin
+          Tempfile.create { HTTParty.get(uri)&.body }
+        rescue Exception => e
+          FCC.error "=> Exception: '#{e}'. Skipping download."
+
+          raise e
+          return false
+        end
+      end
+
+      def lms_date
+        Date.today.strftime('%m-%d-%Y')
+      end
+    end
+  end
+end

data/lib/fcc/station/parsers/extended_info.rb

@@ -0,0 +1,78 @@
+require 'httparty'
+
+module FCC
+  module Station
+    module Parsers
+      class ExtendedInfo < HTTParty::Parser
+        def parse
+          results = []
+          body.each_line do |row|
+            attrs = {}
+            attrs[:raw] = row
+            fields = row.split('|').slice(1...-1).collect(&:strip).map { |v| v == '-' ? "" : v }
+
+            attrs[:call_sign]                = fields[0]
+            attrs[:frequency]                = parse_frequency(fields[1])
+            attrs[:band]                     = fields[2]
+            attrs[:channel]                  = fields[3]
+            attrs[:antenna_type]             = fields[4] # Directional Antenna (DA) or NonDirectional (ND)
+            attrs[:operating_hours]          = fields[5] if fields[5] && attrs[:band]&.upcase == "AM" # (Only used for AM)
+            attrs[:station_class]            = fields[6]
+            attrs[:region_2_station_class]   = fields[7] if fields[7] && attrs[:band]&.upcase == "AM" # (only used for AM)
+            attrs[:status]                   = fields[8]
+            attrs[:city]                     = fields[9]
+            attrs[:state]                    = fields[10]
+            attrs[:country]                  = fields[11]
+            attrs[:file_number]              = fields[12]  #File Number (Application, Construction Permit or License) or
+            attrs[:signal_strength]          = parse_signal_strength(fields[13]) # Effective Radiated Power --
+            attrs[:effective_radiated_power] = parse_signal_strength(fields[14]) # Effective Radiated Power -- vertically polarized (maximum)
+            attrs[:haat_horizontal]          = fields[15] # Antenna Height Above Average Terrain (HAAT) -- horizontal polarization
+            attrs[:haat_vertical]            = fields[16] # Antenna Height Above Average Terrain (HAAT) -- vertical polarization
+            attrs[:fcc_id]                   = fields[17] # Facility ID Number (unique to each station)
+            attrs[:latitude]                 = parse_latitude(fields[18], fields[19], fields[20], fields[21])
+            attrs[:longitude]                = parse_longitude(fields[22], fields[23], fields[24], fields[25])
+            attrs[:licensed_to]              = fields[26] # Licensee or Permittee
+
+            results << attrs
+          end
+
+          results
+        end
+
+        def parse_longitude(direction, degrees, minutes, seconds)
+          decimal_degrees = degrees.to_i + (minutes.to_f / 60) + (seconds.to_f / 3600)
+
+          "#{direction =~ /W/ ? '-' : ''}#{decimal_degrees}"
+        end
+
+        def parse_latitude(direction, degrees, minutes, seconds)
+          decimal_degrees = degrees.to_i + (minutes.to_f / 60) + (seconds.to_f / 3600)
+
+          "#{direction =~ /S/ ? '-' : ''}#{decimal_degrees}"
+        end
+
+        def parse_signal_strength(power_string)
+          return unless power_string.present?
+
+          number, unit = power_string.strip.scan(/^([0-9.]+)\s+(\w+)$?/).flatten
+          multiplier   = case unit&.downcase
+                          when "w"
+                            1
+                          when "kw"
+                            1000
+                          when "mw"
+                            1000000
+                          else
+                            1
+                         end
+      
+          number.to_f * multiplier
+        end
+
+        def parse_frequency(freq)
+          freq.to_f
+        end
+      end
+    end
+  end
+end

data/lib/fcc/station/parsers/lms_data.rb

@@ -0,0 +1,13 @@
+require 'httparty'
+
+module FCC
+  module Station
+    module Parsers
+      class LmsData < HTTParty::Parser
+        def parse
+          body
+        end
+      end
+    end
+  end
+end