wriggler 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d741f12caa9d0cae037e2689e76ff0aa6adee13
-  data.tar.gz: d84a237b2f9c49ef76df0a1fa7db330d32d031ea
+  metadata.gz: b1d083c4ddf26508f6ca9b68f5574bf1491eba46
+  data.tar.gz: 2f347c815b35996afbec369bc0f39eb56cb2c739
 SHA512:
-  metadata.gz: 7fd896183312ea3a2f9ab5e00836dfaf12f7af1486796deedd2f80f1c3f851ae437d29d8c48566a8f89c8ff04148b51c797ad82a8518fd65fb025178f75c80c0
-  data.tar.gz: 0aa7b338518a6285e5bb3aac82ff401b0157a3b72f805b49f2b066851e1aa81f22a7f03924db2625ea50eece17c6492942424664e380fa2c50ab89d5dd0f0b8d
+  metadata.gz: b3fab4d15b405776477b04317e34f757971ab4f6c14cb1041ba2040e513bbcb8a0cc837427372843e77365e322d82ae72f44aa54fbd0c724fa5e224c276b48b9
+  data.tar.gz: 8d8a1aace53b38470d54f0dade3708e0645211aaaa25d6e09f78175bccb93c9cf24af2e44735e3311b929c98ff0a2cf34a9194b26342b063579921a4c896bc99

data/README.md

@@ -1,8 +1,6 @@
 # Wriggler
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/wriggler`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-Wriggler was created to serve and the crawler for a search engine, moving its way through HTML files and grabbing data based on pre determined tags then making/storing the data in a number of specifically created CSV files. 
+Wriggler was created to serve and the crawler for a search engine, moving its way through HTML and/or XML files and grabbing data based on pre determined tags then making/storing the data in a specifically created CSV file. Wriggler acts similarly t0 a spider, but was designed to be used with any number of local files, not as an actual web scraper.
 
 ## Installation
 
@@ -22,18 +20,20 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
-
-## Development
+You only need to run one command to use Wriggler, run: 
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+```ruby
+Wriggler.crawl([array, of, HTML/XML, tags], directory)
+```
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+Note: The directory in this should be the top level directory that your HTML/XML files are in. Wriggler will account for any nested directories within this directory that also contain HTML/XML files and at the end of it running will save a new file named "tag_content.csv" to this directory
 
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/wriggler. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
 
+On top of that, please contribute. I built this for a very specific reason, but I would very much like to see it become something bigger, so if you can assist with that please do!
+
 
 ## License
 

data/lib/wriggler/version.rb

@@ -1,3 +1,3 @@
 module Wriggler
-  VERSION = "0.5.0"
+  VERSION = "1.0.0"
 end

data/lib/wriggler.rb

@@ -92,8 +92,18 @@ module Wriggler
   end
 end
 
+require 'CSV'
+
 module Writer
-	def write(content)
-    @content = content
+	def self.write(content)
+    #Write to a CSV file now
+    column_names = content.keys
+    s = CSV.generate do |csv|
+      csv << column_names
+      content.keys.each do |key|
+        csv << content.fetch(key)
+      end
+    end
+    File.write('tag_content.csv', s)
 	end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wriggler
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Elliott Young
@@ -85,14 +85,8 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- dirtest/nested_fldr/test5.xml
-- dirtest/test1.xml
-- dirtest/test2.xml
-- dirtest/test3.xml
-- dirtest/test4.html
 - lib/wriggler.rb
 - lib/wriggler/version.rb
-- test.rb
 - wriggler.gemspec
 homepage: https://github.com/ElliottAYoung/wriggler
 licenses:

data/dirtest/nested_fldr/test5.xml

@@ -1 +0,0 @@
-<test>If this appears it works</test>

data/dirtest/test1.xml

@@ -1,31 +0,0 @@
-<root>
-  <sitcoms>
-    <sitcom>
-      <test>This is different</test>
-      <name>Married with Children</name>
-      <characters>
-        <character>Al Bundy</character>
-        <character>Bud Bundy</character>
-        <character>Marcy Darcy</character>
-      </characters>
-    </sitcom>
-    <sitcom>
-      <name>Perfect Strangers</name>
-      <characters>
-        <character>Larry Appleton</character>
-        <character>Balki Bartokomous</character>
-      </characters>
-    </sitcom>
-  </sitcoms>
-  <dramas>
-    <drama>
-      <name>The A-Team</name>
-      <characters>
-        <character>John "Hannibal" Smith</character>
-        <character>Templeton "Face" Peck</character>
-        <character>"B.A." Baracus</character>
-        <character>"Howling Mad" Murdock</character>
-      </characters>
-    </drama>
-  </dramas>
-</root>

data/dirtest/test2.xml

@@ -1,30 +0,0 @@
-<root>
-  <sitcoms>
-    <sitcom>
-      <name>Married with Children</name>
-      <characters>
-        <character>Al Bundy</character>
-        <character>Bud Bundy</character>
-        <character>Marcy Darcy</character>
-      </characters>
-    </sitcom>
-    <sitcom>
-      <name>Perfect Strangers</name>
-      <characters>
-        <character>Larry Appleton</character>
-        <character>Balki Bartokomous</character>
-      </characters>
-    </sitcom>
-  </sitcoms>
-  <dramas>
-    <drama>
-      <name>The A-Team</name>
-      <characters>
-        <character>John "Hannibal" Smith</character>
-        <character>Templeton "Face" Peck</character>
-        <character>"B.A." Baracus</character>
-        <character>"Howling Mad" Murdock</character>
-      </characters>
-    </drama>
-  </dramas>
-</root>

data/dirtest/test3.xml

@@ -1,30 +0,0 @@
-<root>
-  <sitcoms>
-    <sitcom>
-      <name>Married with Children</name>
-      <characters>
-        <character>Al Bundy</character>
-        <character>Bud Bundy</character>
-        <character>Marcy Darcy</character>
-      </characters>
-    </sitcom>
-    <sitcom>
-      <name>Perfect Strangers</name>
-      <characters>
-        <character>Larry Appleton</character>
-        <character>Balki Bartokomous</character>
-      </characters>
-    </sitcom>
-  </sitcoms>
-  <dramas>
-    <drama>
-      <name>The A-Team</name>
-      <characters>
-        <character>John "Hannibal" Smith</character>
-        <character>Templeton "Face" Peck</character>
-        <character>"B.A." Baracus</character>
-        <character>"Howling Mad" Murdock</character>
-      </characters>
-    </drama>
-  </dramas>
-</root>

data/dirtest/test4.html

@@ -1,7 +0,0 @@
-<div id = "buttons">
-    <button id="bye">Bye</button>
-    <button id="hello">Hello</button>
-</div>
-<div>
-    <p>1: <span id="greeting">Greeting</span></p>
-</div>

data/test.rb

@@ -1,93 +0,0 @@
-require "nokogiri"
-require "find"
-
-class Wriggler
- def initialize(tags=[], directory="", subdirectories=true)
-    @content = Hash[tags.map {|k| [k, []]}]   #Hash with content
-    @directory = directory                    #Current top-level directory
-    @subdirectories = subdirectories          #Default true for the existence of subdirs
-
-    navigate_directory
-    p @content
-    # Writer.write(@content)
-  end
-
-  private
-
-  def navigate_directory
-    #Set the cwd to the given dir send to gather all nested files from there
-    Dir.chdir(@directory) 
-    open_files(gather_files)
-  end
-
-  def gather_files
-    #Gathers all of the HTML or XML files from this and all subdirectories into an array
-    file_array = []
-    Find.find(@directory) do |f|
-      file_array << f if f.match(/\.xml\Z/) || f.match(/\.html\Z/)
-    end
-    file_array
-  end
-
-  def open_files(file_array)
-    file_array.each do |file|
-      open_next_file(file)
-    end
-  end
-
-  def open_next_file(file)
-    #Opens the next file on the list, depending on the extension passes it to HTML or XML
-    f = File.open(file)
-
-    if is_HTML?(file)
-      set_HTML(f)
-    elsif is_XML?(file)
-      set_XML(f)
-    end
-  end
-
-  def is_HTML?(file)
-    #Determines, using a regex check, if it is an HTML file
-    file =~ /.html/
-  end
-
-  def is_XML?(file)
-    #Determines, using a regex check, if it is an XML file
-    file =~ /.xml/
-  end
-
-  def set_HTML(file)
-    #Set the HTML file into Nokogiri for crawling
-    doc = Nokogiri::HTML(file)
-    crawl_file(doc)
-  end
-
-  def set_XML(file)
-    #Set the XML file into Nokogiri for crawling
-    doc = Nokogiri::XML(file)
-    crawl_file(doc)
-  end
-
-  def crawl_file(doc)
-    #Crawl the Nokogiri Object for the file
-    @content.each_key do |key|
-      arr = []
-      if !doc.xpath("//#{key}").empty?        #Returns an empty array if tag is not present
-        doc.xpath("//#{key}").map{ |tag| arr << sanitize(tag.text) }
-      end
-      fill_content(arr, key)
-    end
-  end
-
-  def sanitize(text)
-    #Removes any escaped quotes, replaces them
-    text.gsub(/"/, "'").lstrip.chomp        
-  end
-
-  def fill_content(arr, key)
-    #Doesn't shovel if there is no content found for the specific tag
-    !arr.empty? ? (@content.fetch(key) << arr) : nil
-  end
-end
-
-test = Wriggler.new(["character", "content", "name", "title", "test"], "/Users/47900/Desktop/Ruby/wriggler/dirtest", false)