sec-firms 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 497947b9cfb3ad25cdfb8a798d976dc10d615a28
+  data.tar.gz: 76a9e3cf60c05f3f9de9fe15e6ace31de6e65571
+SHA512:
+  metadata.gz: b5e7891dc979c5b0df269790b9ec66b6741152e892c37f4bd174b5afa07efa92741918c524b92200708b6db4b7a53d55b6d4e24287147d01520328021a17479b
+  data.tar.gz: 64bf1ff42e6053b78bd881c2e441b97d691cbbe359ccd26062448d423e98847c54ea4bc43ee0701196ce606ca4a364a5cd2b13dd227d910a28c943ca41b0822d

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.0
+script: "bundle exec rake spec"

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sec-firms.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Josemar Luedke
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,82 @@
+# Sec::Firms [![Build Status](https://travis-ci.org/neighborly/sec-firms.svg?branch=master)](https://travis-ci.org/neighborly/sec-firms)
+
+Firms XML parser for the U. S. Securities and Exchange Commission
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sec-firms'
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+## Usage
+
+SEC maintains a list of the firms with the CIK ID that is updated every month. We have an API to access that list as csv with the following:
+
+```ruby
+Sec::Firms::Lists.new.latest_as_csv
+```
+
+To fetch all the data avaliable from a firm, you can use the following:
+
+```ruby
+Sec::Firms::FirmParser.new(cik_id).to_hash
+```
+
+Here's an exaple using both classes:
+
+```ruby
+firms = CSV.read(Sec::Firms::Lists.new.latest_as_csv)[3..-1]
+
+firms.each_with_index do |row, index|
+  cik_id = row[2].strip
+  firm = Sec::Firms::FirmParser.new(cik_id).to_hash
+
+  json = JSON.pretty_generate(firm.as_json)
+  file_name = Rails.root.join("data/#{cik_id}-#{firm['slug']}.json")
+  File.write(file_name, json)
+
+  puts "#{index + 1} out of #{firms.count}"
+end
+```
+
+## Configuration
+
+```ruby
+Sec::Firms.configure do |config|
+  config.root_path = "#{Rails.root}/data"
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/sec-firms/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## License
+
+Licensed under the [MIT license](LICENSE.txt).
+
+## About Neighborly
+
+[![Neighborly](https://cloud.githubusercontent.com/assets/230476/10037857/285b76f6-6171-11e5-9af9-3ce60861e8c8.png)](https://neighborly.com)
+
+**Neighborly is the nation's first Community Investment Marketplace.™**
+
+By providing better ways for people to invest directly in the places and civic projects they care about, we create new options for communities to approach civic capital formation. Neighborly is democratizing the $3.8T municipal securities market, fostering a healthier relationship between global banks and our nation's places.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sec/firms"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/sec/firms/configuration.rb

@@ -0,0 +1,7 @@
+module Sec
+  module Firms
+    class Configuration
+      attr_accessor :root_path
+    end
+  end
+end

data/lib/sec/firms/downloader.rb

@@ -0,0 +1,44 @@
+require 'net/http'
+require 'uri'
+require 'digest/md5'
+require 'fileutils'
+
+module Sec
+  module Firms
+    class Downloader
+      attr_reader :url, :root_path
+
+      def initialize(url)
+        @url = url
+        @root_path = "#{Sec::Firms.configuration.root_path}/downloads/"
+      end
+
+      def content
+        if File.exist?(file_name)
+          read_file
+        else
+          save_to_file
+        end
+      end
+
+      def file_name
+        "#{root_path}#{Digest::MD5.hexdigest(url)}.xml"
+      end
+
+      private
+
+      def read_file
+        IO.read(file_name)
+      end
+
+      def save_to_file
+        request = Net::HTTP.get_response(URI.parse(url))
+        if request.code == '200'
+          FileUtils.mkdir_p(root_path)
+          File.write(file_name, request.body)
+          request.body
+        end
+      end
+    end
+  end
+end

data/lib/sec/firms/firm_entry_parser.rb

@@ -0,0 +1,30 @@
+module Sec
+  module Firms
+    class FirmEntryParser < XMLParser
+      attr_reader :updated_at
+
+      def initialize(xml_str, updated_at)
+        super(xml_str) if xml_str
+        @updated_at = updated_at
+      end
+
+      def to_hash
+        data = Hash.from_xml(doc.to_s)['edgarSubmission']['formData'] rescue nil
+
+        return {} unless data
+        data = Helpers::transform_hash(data) do |hash, key, value|
+          hash[key] = default_values(value)
+        end
+      end
+
+      private
+
+      def default_values(value)
+        {
+          value: value,
+          updated_at: updated_at
+        }
+      end
+    end
+  end
+end

data/lib/sec/firms/firm_parser.rb

@@ -0,0 +1,83 @@
+module Sec
+  module Firms
+    class FirmParser < XMLParser
+      SEC_FIRM_DETAIL_FILE_NAME = 'primary_doc.xml'
+      SEC_FIRM_URL = 'http://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=:CIK_ID&output=atom'
+
+      def initialize(cik_id)
+        fetch_url = SEC_FIRM_URL.gsub(':CIK_ID', cik_id)
+        xml_str = Net::HTTP.get(URI.parse(fetch_url))
+        super(xml_str)
+      end
+
+      def id
+        doc.xpath('//xmlns:cik').text
+      end
+
+      def slug
+        Slugify.convert(name[:value], true)
+      end
+
+      def name
+        default_values doc.xpath('//xmlns:conformed-name').text
+      end
+
+      def to_hash
+        data = Hash.from_xml(doc.to_s)['feed']['company_info']
+        data = Helpers::transform_hash(data) do |hash, key, value|
+          hash[key.underscore] = default_values(value)
+        end
+
+        {
+          id: id,
+          slug: slug,
+          name: name
+        }.merge(data).merge(entries_attributes)
+          .deep_transform_keys { |key| key.to_s.underscore }
+      end
+
+      def entries
+        doc.xpath('//xmlns:entry').map do |entry|
+          filing_date = Date.parse(entry.at('filing-date').text)
+          accession_number = entry.at('accession-nunber').text
+          [
+            [filing_date, accession_number],
+            entry
+          ]
+        end.sort_by { |value| value[0] }
+      end
+
+      private
+
+      def updated_at
+        @updated_at ||= doc.xpath('//xmlns:feed').at('updated').text
+      end
+
+      def default_values(value)
+        {
+          value: value,
+          updated_at: updated_at
+        }
+      end
+
+      def entries_attributes
+        hash = {}
+        details = entries.map do |entry|
+          FirmEntryParser.new(
+            Downloader.new(entry_url(entry[1])).content, entry[0][0]
+          ).to_hash
+        end
+
+        details.each do |detail|
+          hash.merge!(detail)
+        end
+        hash
+      end
+
+      def entry_url(entry)
+        url = entry.at('filing-href').text
+        url.gsub(url.split('/')[-1], SEC_FIRM_DETAIL_FILE_NAME)
+      end
+    end
+  end
+end

data/lib/sec/firms/helpers.rb

@@ -0,0 +1,17 @@
+module Sec
+  module Firms
+    module Helpers
+      def self.transform_hash(original, options = {}, &block)
+        original.inject({}) do |result, (key, value)|
+          value = if (options[:deep] && Hash === value)
+                    transform_hash(value, options, &block)
+                  else
+                    value
+                  end
+          block.call(result, key, value)
+          result
+        end
+      end
+    end
+  end
+end

data/lib/sec/firms/lists.rb

@@ -0,0 +1,70 @@
+require 'roo-xls'
+require 'zip'
+
+module Sec
+  module Firms
+    class Lists < XMLParser
+      URL = 'https://www.sec.gov/foia/docs/muniadvisors-archive.htm'
+
+      def initialize
+        xml = Net::HTTP.get(URI.parse(URL))
+        super(xml)
+      end
+
+      def latest_as_csv
+        convert_to_csv(unzip(latest_report).first)
+      end
+
+      def urls
+        doc.css('table:last li a').map do |anchor|
+          [
+            Date.parse(anchor.text),
+            expand_url(anchor.attr('href'))
+          ]
+        end.sort.reverse
+      end
+
+      private
+
+      def latest_report
+        url = urls.first[1]
+        full_path = "/tmp/#{Digest::MD5.hexdigest(url)}.zip"
+
+        unless File.exist?(full_path)
+          remote_file = open(url)
+          File.open(full_path, 'wb') do |file|
+            file.write(remote_file.read)
+          end
+        end
+
+        full_path
+      end
+
+      def unzip(zip_path)
+        Zip::File.open(zip_path) do |zip_file|
+          zip_file.map do |entry|
+            file_path = File.join(File.dirname(zip_path), entry.name)
+            entry.extract(file_path) unless File.exist?(file_path)
+            file_path
+          end
+        end
+      end
+
+      def convert_to_csv(xls_path)
+        csv_path = File.join(File.dirname(xls_path),
+                             File.basename(xls_path) + '.csv')
+        xls = Roo::Excel.new(xls_path)
+        xls.to_csv(csv_path)
+        csv_path
+      end
+
+      def latest_report_url
+        doc.css('table:nth-child(2) li a').first.attribute('href')
+      end
+
+      def expand_url(relative_path)
+        URI.join(URL, relative_path).to_s
+      end
+    end
+  end
+end

data/lib/sec/firms/version.rb

@@ -0,0 +1,5 @@
+module Sec
+  module Firms
+    VERSION = "0.1.0"
+  end
+end

data/lib/sec/firms/xml_parser.rb

@@ -0,0 +1,13 @@
+require 'nokogiri'
+
+module Sec
+  module Firms
+    class XMLParser
+      attr_accessor :doc
+
+      def initialize(xml_str)
+        @doc = Nokogiri::XML(xml_str, nil)
+      end
+    end
+  end
+end

data/lib/sec/firms.rb

@@ -0,0 +1,25 @@
+require 'slugify'
+require 'sec/firms/version'
+require 'sec/firms/helpers'
+require 'sec/firms/configuration'
+require 'sec/firms/downloader'
+require 'sec/firms/xml_parser'
+require 'sec/firms/firm_entry_parser'
+require 'sec/firms/firm_parser'
+require 'sec/firms/lists'
+
+module Sec
+  module Firms
+    class << self
+      attr_writer :configuration
+    end
+
+    def self.configuration
+      @configuration ||= Configuration.new
+    end
+
+    def self.configure
+      yield(configuration)
+    end
+  end
+end

data/sec-firms.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sec/firms/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sec-firms"
+  spec.version       = Sec::Firms::VERSION
+  spec.authors       = ['Neighborly' 'Irio Musskopf', 'Josemar Luedke']
+  spec.email         = %w(howdy@neighborly.com iirineu@gmail.com josemarluedke@gmail.com)
+
+  spec.summary       = "Firms parser for sec.gov"
+  spec.description   = "Firms parser for the U. S. Securities and Exchange Commission"
+  spec.homepage      = "https://github.com/neighborly/sec-firms"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "roo-xls", "~> 1.0"
+  spec.add_dependency "rubyzip", "~> 1.1"
+  spec.add_dependency "slugify", "~> 1.0"
+  spec.add_dependency "nokogiri", "> 1.5"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.3.0"
+end

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: sec-firms
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- NeighborlyIrio Musskopf
+- Josemar Luedke
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: roo-xls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: slugify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+description: Firms parser for the U. S. Securities and Exchange Commission
+email:
+- howdy@neighborly.com
+- iirineu@gmail.com
+- josemarluedke@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/sec/firms.rb
+- lib/sec/firms/configuration.rb
+- lib/sec/firms/downloader.rb
+- lib/sec/firms/firm_entry_parser.rb
+- lib/sec/firms/firm_parser.rb
+- lib/sec/firms/helpers.rb
+- lib/sec/firms/lists.rb
+- lib/sec/firms/version.rb
+- lib/sec/firms/xml_parser.rb
+- sec-firms.gemspec
+homepage: https://github.com/neighborly/sec-firms
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Firms parser for sec.gov
+test_files: []