lac 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ca244cdc01499a1bae292429eafed751e4555e0a
+  data.tar.gz: f44e2769c311b844e0d12656fc2697b0a6eba6ee
+SHA512:
+  metadata.gz: c71bdd27d36a4f374ab624b12f0c40c6ae7a91a6fa8815fd13691ffa597b8a4c967f0ab3e4a46e26afb7b23e4bbc9834f148ab92cc76cc27681a2f59d347387f
+  data.tar.gz: 7a80d9f99c59034a2c647da55cd7e34205d0dd94b56f4bc9e92c7c8987ff22dac4546f0e334e6decda4b53231dc1bc7ac560cf860f32f200d59582d5ed187a1e

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/bin/lac

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lac"
+
+Lac::CLI.new(ARGV).start

data/lib/lac.rb

@@ -0,0 +1,9 @@
+module Lac
+
+end
+
+require "lac/page"
+require "lac/base_model"
+require "lac/application"
+require "lac/base"
+require "lac/cli"

data/lib/lac/application.rb

@@ -0,0 +1,98 @@
+require 'yaml'
+require 'haml'
+
+require 'active_support/all'
+
+require 'open-uri'
+require 'uri'
+require 'csv'
+
+require 'lac'
+require 'sinatra'
+require "sinatra/base"
+
+module Lac
+  class Application < Sinatra::Base
+
+    def render_file(file, params= {})
+      gemdir = Gem.loaded_specs["lac"].gem_dir
+      base_template = File.read(gemdir + "/views/base.html.haml")
+      content = Haml::Engine.new(File.read(gemdir + "/views/#{file}.html.haml")).render(Object.new, models: @@models, scrapers: @@scrapers, **params)
+      return Haml::Engine.new(base_template).render(Object.new, content: content, models: @@models, scrapers: @@scrapers)
+    end
+
+    model_files =  Dir["./models/*"]
+    helper_files =  Dir["./helpers/*"]
+    scraper_files =  Dir["./scrapers/*"]
+
+    data = {}
+
+    @@scrapers = []
+    @@models = []
+
+    model_files.each do |filename|
+      require filename
+      puts filename
+      @@models << Object.const_get(filename.split("/").last.split(".").first.camelize)
+      puts Object.const_get(filename.split("/").last.split(".").first.camelize)
+    end
+
+    helper_files.each do |filename|
+      require filename
+      puts filename
+    end
+
+    scraper_files.each do |filename|
+      require filename
+      puts filename
+      @@scrapers << Object.const_get(filename.split("/").last.split(".").first.camelize)
+      Object.const_get(filename.split("/").last.split(".").first.camelize).data
+    end
+
+
+
+
+    ## System routes
+
+    get '/' do
+      render_file("home")
+    end
+
+    get '/models' do
+      render_file("models")
+    end
+
+    get '/scrapers' do
+      render_file("scrapers")
+    end
+
+    get '/scrapers/:name/scrape' do
+      scraper = @@scrapers.select { |scraper| params[:name] == scraper.name.underscore }.first
+      scraper.data
+      render_file("scrapers")
+    end
+
+
+    get '/models/:name/csv' do
+
+      content_type 'application/csv'
+      attachment "myfilename.csv"
+      m = @@models.select { |model| params[:name] == model.name.underscore }.first
+      csv_string = CSV.generate(col_sep: ";") do |csv|
+          header_row = ["id"] + m.fields
+          m.all.each do |id, obj|
+            csv << [id] + m.fields.map {|a| obj[a]}
+          end
+      end    
+      # render_file("", { })
+    end
+
+    get '/models/:name' do
+      render_file("model", {model: @@models.select { |model| params[:name] == model.name.underscore }.first })
+    end
+
+
+    puts "Loaded Models #{@@models}"
+
+  end
+end

data/lib/lac/base.rb

@@ -0,0 +1,4 @@
+module Lac
+  class Base
+  end
+end

data/lib/lac/base_model.rb

@@ -0,0 +1,77 @@
+require 'yaml'
+module Lac
+
+  class BaseModel
+    
+    # Any suggestions on how to manage these fields 
+    # better are welcome!
+    
+    @@fields = []
+    
+    def initialize
+    end
+
+    def validates?
+      return true unless primary_key
+      self.method(primary_key).call.present?
+    end
+
+    def self.field(symbol)
+
+      @@fields.push(symbol)
+      @@fields = @@fields.uniq
+      attr_accessor symbol
+    end
+   
+    def to_hash
+      hash = {}
+      @@fields.each do |field|
+        if field
+          hash[field] = self.method(field).call
+        end
+      end
+      hash
+    end
+    
+    def self.fields
+      self.new.fields
+    end
+    
+    def self.primary_key
+      self.new.primary_key
+    end
+
+    def self.all
+      begin
+        YAML::load_file("#{self.name}.yml")
+        YAML::load_file("#{self.name}.yml")[:collecion]
+        YAML::load_file("#{self.name}.yml")[:collection]
+      rescue
+        []
+      end
+    end
+
+    def fields
+      @@fields
+    end
+
+    def save
+      filename = "#{self.class}.yml"
+      FileUtils.touch(filename)
+      if validates? && id
+        collection = YAML::load_file(filename)
+        collection = {} unless collection
+        collection[:collection] = {} unless collection[:collection]
+        collection[:collection][id.to_sym] = to_hash
+        File.open(filename, 'w') {|f| f.write collection.to_yaml }
+      end
+    end
+
+    def id
+      if self.validates?
+        Digest::SHA256.hexdigest(self.method(self.primary_key).call)
+      end
+    end
+
+  end
+end

data/lib/lac/cli.rb

@@ -0,0 +1,96 @@
+require 'colorize'
+require 'readline'
+require 'lac'
+require 'nokogiri'
+# require './application.rb'
+
+
+module Lac
+  class CLI
+
+      def initialize(arguments)
+        @action = arguments[0]
+      end
+      
+      def start
+        if @action == "new"
+          create_project
+        elsif @action == "server"
+          start_server
+        else
+          puts "Invalid command, exiting".red.on_white.bold
+          puts "The only valid command is 'new'".green.on_white.bold
+        end
+        exit_cli
+      end
+
+      def start_server
+        Lac::Application.run!
+      end
+
+      def create_project
+          puts "Hello! Welcome to Lac".bold
+          puts "This is a work in progress. Check out the Git repo if you would like to contribute.".bold
+          puts "Type 'exit' to quit"
+          project_name = nil
+          
+          while project_name == nil
+            input = prompt_input("Choose a name for your project: ")
+            if File.directory?("./#{input}")
+              puts "A folder with that name already exists, choose another name:"
+            else
+              project_name = input
+            end
+          end
+           
+          `mkdir #{project_name}`
+          if File.directory?("./#{project_name}")
+            puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/"
+            
+            `mkdir #{project_name}/config`
+            puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/config"
+
+            `mkdir #{project_name}/cache`
+            puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/cache"
+
+            `mkdir #{project_name}/models`
+            puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/models"
+
+            `mkdir #{project_name}/scrapers`
+            puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/scrapers"
+
+            `mkdir #{project_name}/helpers`
+            puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/helpers"
+          else
+            puts "Something went wrong. Exiting...".red.bold
+            exit_cli
+          end
+
+
+          File.open("./#{project_name}/Gemfile", "w") do |f|
+            f.puts "source 'https://rubygems.org'"
+            f.puts "gem 'lac'"
+          end
+          puts "#{"[Success]".green.bold} created file!\t\t ./#{project_name}/Gemfile"
+
+          File.open("./#{project_name}/config/lac.rb", "w") do |f|
+            f.puts ""
+          end
+          puts "#{"[Success]".green.bold} created file!\t\t ./#{project_name}/config/lac.rb"
+      end
+
+      def prompt_input(message)
+          input = Readline.readline(message, true).strip
+          exit_cli if input.downcase === "exit"
+          return input.to_s
+      end
+
+      def exit_cli
+          puts "#{"\nCya!".yellow.on_black.bold}\n"
+          exit
+      end
+  end
+end
+
+
+# puts Lac::CLI.new.start

data/lib/lac/page.rb

@@ -0,0 +1,77 @@
+require 'nokogiri'
+
+module Lac
+
+  class Page
+
+    attr_accessor :html
+
+    # Need better flow for what this does.
+
+    def try_css_attr(css, attr)
+      if element = self.try_css(css)
+        element.attr(attr)
+      else
+        nil
+      end
+    end
+
+    def try_css_parent_attr(css, attr)
+      if element = try_css(css)
+        element.parent.attr(attr)
+      else
+        nil
+      end
+    end
+
+    def try_css(css)
+      self.html.css(css).first
+    end
+
+
+    # Simply gets a webpage based on a url.
+    # from_cache = true (default) will take a cached version
+    # if it exists.
+
+    def self.get_page(url, from_cache = true)
+      url_hash = Digest::SHA256.hexdigest(url)
+      filename = "cache/#{url_hash}"
+      if from_cache && File.file?(filename)
+        result = open(filename).read
+        puts "Gotten #{filename} from cache"
+      else
+        result = open(url).read
+        File.write(filename, result)
+        puts "Written cache file #{filename}"
+      end
+      return result
+    end
+
+    def initialize html: nil
+      self.html = html
+    end
+
+    # helpers for seamless initialisation no matter what starting point
+    
+    def self.by_url(url)
+      self.new(html: Nokogiri::HTML(get_page(url)))
+    end
+
+    def self.by_html_string(html_string)
+      self.new(html: Nokogiri::HTML(html_string))
+    end
+
+    def self.by_html(html)
+      self.new(html: html)
+    end
+
+
+    # Returns a collection of pages based on a selector.
+    # Use to collect a collection of elements from a page.
+    def collection_by_selector(selector)
+      self.html.css(selector).map{|item| Lac::Page.by_html(item)}
+    end
+
+  end
+
+end

data/views/base.html.haml

@@ -0,0 +1,19 @@
+!!!
+%html
+  %head
+    %meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/
+    %title Lac
+    :css
+      td, th {
+        padding: 1em;
+        background-color: #efefff;
+      }
+      th {
+        background-color: #aeaeaf;
+      }
+  %body
+    %h1 Lac
+    %ul
+      %a{href: "/scrapers"}= "Scrapers (#{scrapers.count})"
+      %a{href: "/models"}= "Models (#{models.count})"
+    = content

data/views/home.html.haml

@@ -0,0 +1 @@
+%p 

data/views/model.html.haml

@@ -0,0 +1,18 @@
+%h1= "Model #{model.name}"
+
+%table
+  %tbody
+    %th Primary Key
+    %td= model.primary_key
+%a{href: "/models/#{model.name.underscore}/csv"} Export to CSV
+%table
+  %thead
+    %th ID
+    - model.fields.each do |field|
+      %th= field
+  %tbody
+    - model.all.each do |id, obj|
+      %tr
+        %td= id
+        - model.fields.each do |field|
+          %td= obj[field.to_sym]

data/views/models.html.haml

@@ -0,0 +1,15 @@
+
+%h1 Models
+
+%table
+  %thead
+    %th Name
+    %th #
+    %th Actions
+  %tbody
+    - models.each do |model|
+      %tr
+        %td= model
+        %td= model.all.count
+        %td
+          %a{href: "/models/#{model.name.underscore}"} Show

data/views/scrapers.html.haml

@@ -0,0 +1,11 @@
+
+%h1 Scrapers
+
+%table
+  %thead
+    %th Name
+  %tbody
+    - scrapers.each do |scraper|
+      %tr
+        %td= scraper
+        

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: lac
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Rene van Pelt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-12-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fileutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: haml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables:
+- lac
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- bin/lac
+- lib/lac.rb
+- lib/lac/application.rb
+- lib/lac/base.rb
+- lib/lac/base_model.rb
+- lib/lac/cli.rb
+- lib/lac/page.rb
+- views/base.html.haml
+- views/home.html.haml
+- views/model.html.haml
+- views/models.html.haml
+- views/scrapers.html.haml
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: lac is a DRY framework for creating scrapers
+test_files: []