geturl 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 411f93bbcd2d3d670c3690bfe90c7bcf90b2af1c4ccab49c79cb0c697245ebc4
+  data.tar.gz: 53246132b245beb548e09311cb291d8dbce585fcaef3b98d3b6356d24ad42353
+SHA512:
+  metadata.gz: 2fb81625dc50caa479849145ebb598d980d5948237c90b9394291a2ed0fd5a8b609ca59d0d84f42eda6388f10e25192703c5e3e6beed3c727dff8f7b3551dd0c
+  data.tar.gz: f97fe1082a7a68ebdd9e1ba91a955d99767e773a9f23f0668db3c772c2e5545ea7d84831abc9be219c0c98a88af32b7c296ceb093c6c0cd0d017b4c26f5dba75

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Taco Jan Osinga
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/bin/geturl

@@ -0,0 +1,199 @@
+#!/usr/bin/env ruby
+
+require 'cli-parser'
+require_relative '../lib/geturl'
+require 'mustache'
+
+module GetUrlCli
+  extend self
+
+  # Reads the command line and runs the selected action.
+  def run
+    flags = %w(--save --delete --reload --sources --register --unregister --reload -v --show-all --color)
+    parameters = %w(--format --sources --tags --description --template)
+    @arguments, @options = CliParser.parse(flags, parameters)
+
+    if @options.include?('--save')
+      save
+    elsif @options.include?('--delete')
+      delete
+    elsif @options.include?('--reload')
+      reload
+    elsif @options.include?('--sources')
+      sources
+    elsif @options.include?('--register')
+      register
+    elsif @options.include?('--unregister')
+      unregister
+    elsif @options.include?('--show-all')
+      show_all
+    elsif !@arguments.empty?
+      search
+    else
+      show_usage
+    end
+  end
+
+  # Outputs how to use this tool.
+  def show_usage
+    puts %q{
+    # Searching
+    geturl keyword1 keyword2 ...
+
+    geturl --format (json|yaml|mustache) keyword1 keyword2
+    geturl --source [source] keyword1 keyword2
+
+    # Local bookmarked urls
+    geturl --save http://www.example.com "Example website" --tags tag1,tag2 --description "This is an example"
+    geturl --delete http://www.example.com
+
+    # Read-only bookmarks from external sources
+    geturl --sources
+    geturl --register [source-id] [file-or-url]
+    geturl --unregister [source-id]
+    geturl --show-all [source-id]
+    geturl --reload
+
+    }.gsub(/^    /, '')
+  end
+
+  # Stores a url locally.
+  def save
+    check_arguments(%w(url name), 'save')
+    url = @arguments[0]
+    name = @arguments[1].gsub(/^"/, "").gsub(/"$/, "")
+
+    GetUrl::LocalUrls.save_url(url,
+                               name,
+                               'description' => @options['--description'].to_s.gsub(/^"/, "").gsub(/"$/, ""),
+                               'tags' => @options['--tags'].to_s.gsub(/^"/, "").gsub(/"$/, "").split(','))
+  end
+
+  # Deletes a locally stored url.
+  def delete
+    check_arguments(%w(url), 'delete')
+    url = @arguments[0]
+    GetUrl::LocalUrls.delete_url(url)
+  end
+
+  # Reloads all sources.
+  def reload
+    id = @arguments[0]
+    GetUrl::Sources.reload(id, verbose: true)
+  end
+
+  # Lists all sources.
+  def sources
+    output_data(GetUrl::Sources.get_sources)
+  end
+
+  # Registers the given source.
+  def register
+    check_arguments(%w(id url), 'register')
+    id = @arguments[0].downcase
+    url = @arguments[1]
+
+    if id == 'local'
+      puts "The source id 'local' is reserved."
+      exit 3
+    end
+
+    print "Registering #{id} ...\r"
+    GetUrl::Sources.register_source(id, url)
+    puts "Registered #{id}" + "".ljust(5)
+    GetUrl::Sources.reload(id, url, verbose: true)
+  end
+
+  # Checks whether the arguments fit the required size, based on given parameter names.
+  def check_arguments(names, action)
+    i = 0
+    blocked_names = ''
+    names.each {|name| blocked_names += "[#{name}] "}
+    names.each {|name|
+      argument = @arguments[i]
+      if argument.to_s.empty?
+        puts "Missing #{name}"
+        puts ''
+        puts "Usage: geturl --#{action} #{blocked_names}"
+        exit 1
+      end
+      i += 1;
+    }
+  end
+
+  # Unregister the given source.
+  def unregister
+    check_arguments(%w(id), 'unregister')
+
+    id = @arguments[0]
+    unless GetUrl::Sources.get_sources.keys.include?(id)
+      puts "Unknown id #{id}"
+      exit 2
+    end
+
+    GetUrl::Sources.unregister_source(id)
+    puts "Removed #{id}"
+  end
+
+  # Shows all urls.
+  def show_all
+    output_data(GetUrl::Searcher.all(@arguments))
+  end
+
+  # Searches for specific urls.
+  def search
+    ids = @option['--sources'].to_s.split(',') rescue nil
+    items = GetUrl::Searcher.search(@arguments, 'ids' => ids)
+    output_data(items)
+  end
+
+  # Outputs the given data object in the selected format.
+  def output_data(data)
+    format = @options['--format'].to_s.downcase
+    if format == 'json'
+      puts JSON.pretty_generate(data)
+    elsif format == 'yaml'
+      puts data.to_yaml
+    elsif format == 'mustache'
+      template = @options['--template'].to_s.gsub(/^"/, "").gsub(/"$/, "")
+      if template.empty?
+        puts 'Missing option: --template'
+        exit 4
+      end
+      data = {items: data} if data.is_a?(Array)
+      template = File.read(template.gsub(/^@/, '')) if template.start_with?('@')
+      puts Mustache.render(template, data)
+    elsif data.is_a?(Hash)
+      data.each {|key, value|
+        puts key.ljust(20) + value.to_s
+      }
+    elsif data.is_a?(Array)
+      data.each {|value|
+        if value.is_a?(Hash) && !value['url'].empty? && !value['name'].empty?
+          output_item(value)
+        else
+          puts value
+        end
+      }
+    else
+      puts data
+    end
+
+  end
+
+  # Outputs the given item as text.
+  def output_item(item)
+    puts item['name']
+    puts item['url']
+    puts item['description'] unless item['description'].to_s.empty?
+    puts 'Tags: ' + item['tags'].join(', ') unless item['tags'].empty?
+    puts ''
+  end
+
+end
+
+GetUrlCli.run
+
+
+
+

data/lib/geturl.rb

@@ -0,0 +1,5 @@
+$:.unshift(File.expand_path(File.dirname(__FILE__)))
+
+require_relative 'geturl/geturl-sources'
+require_relative 'geturl/geturl-local-urls'
+require_relative 'geturl/geturl-searcher'

data/lib/geturl/geturl-file-manager.rb

@@ -0,0 +1,72 @@
+require 'yaml'
+
+module GetUrl
+
+  # The FileManager deals with the local (cached) files.
+  module FileManager
+    extend self
+
+    # Ensure the sources folder exists.
+    @sources_path = File.join(Dir.home, '.geturl', 'sources')
+    FileUtils.mkdir_p(@sources_path)
+
+    # Loads the items from the given local file.
+    #
+    # @param [String]  file   The path of the file
+    # @return [Array]         The items
+    def load_items_from_yaml_file(file)
+      str = File.read(file) rescue ''
+      return load_items_from_yaml_string(str)
+    end
+
+
+    # Loads the items from the given string.
+    #
+    # @param [String]  str    The string
+    # @return [Array]
+    def load_items_from_yaml_string(str)
+      items = []
+      yaml = YAML.load(str).to_a || [] rescue []
+      yaml.each {|raw|
+        items << {
+            'url' => raw['url'],
+            'name' => raw['name'],
+            'description' => raw['description'],
+            'tags' => raw['tags'].to_a
+        } if (raw.include?('url')) && (raw.include?('name'))
+      }
+      return items
+    end
+
+    # Save the given items to the given file in Yaml.
+    #
+    # @param [Array]    items An array of items
+    # @param [String]   file  The path of the file
+    # @return [Integer]
+    def save_items_to_yaml_file(items, file)
+      str = "# This file is generated by geturl.\n"
+      str += items.to_yaml
+      File.write(file, str)
+    end
+
+    # Loads the clean items in the given Yaml string and stores it in the given file
+    #
+    # @param [Object]  str  The Yaml string
+    # @param [Object]  file The path of the file
+    # @return [Integer]
+    def clean_and_save_items_to_yaml_file(str, file)
+      cleaned_items = load_items_from_yaml_string(str)
+      save_items_to_yaml_file(cleaned_items, file)
+    end
+
+    # Returns the local file name for the given source id.
+    #
+    # @param [Object]  id   The source id.
+    # @return [String]      The path of the local source file.
+    def get_local_source_file(id)
+      return File.join(@sources_path, "#{id}.yaml")
+    end
+
+  end
+
+end

data/lib/geturl/geturl-local-urls.rb

@@ -0,0 +1,57 @@
+module GetUrl
+
+  # The LocalUrls manages the local stored urls.
+  module LocalUrls
+    extend self
+
+    @items = nil
+    @private_source_id = 'local'
+    @local_urls_file = File.join(Dir.home, '.geturl', 'sources', 'local.yaml')
+
+    attr_reader :private_source_id
+
+    # Loads all urls items from the local storage.
+    def load_urls
+      @items = FileManager.load_items_from_yaml_file(@local_urls_file) rescue []
+    end
+
+    # Stores all urls ites to the local storage.
+    def save_urls
+      FileManager.save_items_to_yaml_file(@items, @local_urls_file)
+      return @items.size
+    end
+
+    # Adds or updates the data for the given url, and stores in the local storage.
+    #
+    # @param [String]  url     The url.
+    # @param [String]  name    The name.
+    # @param [Hash]  options The options, which might contains the tags and description.
+    def save_url(url, name, options = {})
+      return if (url.to_s.empty?) || (name.to_s.empty?)
+
+      load_urls
+      @items.delete_if {|item| item['url'] == url}
+      options['tags'] ||= []
+      options['tags'].map! {|tag| tag.strip}
+
+      @items << {
+          'url' => url,
+          'source' => 'local',
+          'name' => name,
+          'description' => options['description'],
+          'tags' => options['tags']
+      }
+      save_urls
+    end
+
+    # Deletes the given url from the local storage.
+    #
+    # @param [String]  url  The url.
+    def delete_url(url)
+      load_urls if @urls.nil?
+      @items.delete_if {|item| item['url'] == url}
+      save_urls
+    end
+  end
+
+end

data/lib/geturl/geturl-searcher.rb

@@ -0,0 +1,123 @@
+require 'json'
+
+require_relative 'geturl-file-manager'
+require_relative 'geturl-local-urls'
+
+module GetUrl
+
+  # The Searcher deals with collecting all urls and searching through them.
+  module Searcher
+    extend self
+
+    @items = []
+
+    # Loads the urls of the source with the given id.
+    # @param [String]  id   The id
+    def load_source(id)
+      file = FileManager.get_local_source_file(id)
+      source_items = FileManager.load_items_from_yaml_file(file)
+      source_items.each {|item|
+        item['source'] = id
+        @items << item
+      }
+    end
+
+    # Loads all the given source ids. If ids is empty, all sources will be loaded.
+    # @param [Array]  ids  An array of source ids.
+    def load_sources(ids = nil)
+      ids = Sources.get_sources.keys + ['local'] if ids.nil? || ids.empty?
+      @items.clear
+      load_source(LocalUrls.private_source_id) if ids.include?('local')
+      Sources.get_sources.keys.each {|id|
+        load_source(id) if ids.include?(id)
+      }
+    end
+
+    # Returns all urls of the given ids. If ids is empty, all urls of all sources will be loaded.
+    #
+    # @param [Array]  ids   The array of source ids.
+    # @return [Array]       The array of url items.
+    def all(ids = nil)
+      load_sources(ids)
+      return @items
+    end
+
+    # Searches in all urls for those that matches one or more keywords.
+    # Each item gets a score on relevance.
+    # For specific ids, the options['ids'] can be given.
+    #
+    # @param [Array]  keywords  The keywords
+    # @param [Hash]   options   The options
+    # @return [Array]           The array of items.
+    def search(keywords = [], options = {})
+      load_sources(options['ids'])
+
+      # Calculate score for each item, for each keyword.
+      @items.each {|item|
+        keywords.each {|keyword|
+          add_score(keyword, item)
+        }
+        # Not only individual keywords, but also check on all keywords combined (when mo)
+        if (item.size > 1)
+          full = keywords.join(' ')
+          add_score(full, item, 2)
+        end
+      }
+
+      # Removes all items that are not relevant (score of 0).
+      @items.delete_if {|item| item['score'] == 0}
+
+      # Sort by score
+      @items.sort {|x, y|
+        y['score'] <=> x['score'] rescue y <=> x
+      } if (@items.size > 1)
+
+      # Remove all score values
+      @items.each {|item|
+        item.delete('score')
+      }
+      return @items
+    end
+
+
+    # Adds a score for the given keyword on the given item. The given factor gives extra weight to the score.
+    #
+    # @param [String]   keyword   The keyword.
+    # @param [Hash]     item      The item.
+    # @param [FixNum]   factor    The weighing factor.
+    def add_score(keyword, item, factor = 1)
+      score = 0
+      begin
+        score += get_field_score(item['name'], keyword, 50, 25)
+        score += get_field_score(item['description'], keyword, 10, 2)
+        item['tags'].to_a.each {|tag|
+          score += get_field_score(tag, keyword, 30, 5)
+        }
+      rescue
+      end
+      item['score'] ||= 0
+      item['score'] += score * factor
+    end
+
+
+    # Returns the relevant score:
+    # - If the given value and keyword are equal, the given equal_score is returned.
+    # - Else if the given keyword is a substring of value, the contains_score is returned.
+    # - Otherwise, 0 is returned.
+    #
+    # @param [String]  value            The value.
+    # @param [String]  keyword          The keyword.
+    # @param [Integer]  equal_score     The score when value is equal to the keyword.
+    # @param [Integer]  contains_score  The score when value contains the keyword.
+    # @return [Integer]
+    def get_field_score(value, keyword, equal_score, contains_score)
+      v = value.downcase
+      k = keyword.downcase
+      return equal_score if (v == k)
+      return contains_score if v.include?(k)
+      return 0
+    end
+
+  end
+
+end

data/lib/geturl/geturl-sources.rb

@@ -0,0 +1,87 @@
+require 'yaml'
+require 'fileutils'
+require 'open-uri'
+
+require_relative 'geturl-file-manager'
+
+module GetUrl
+
+  # The Sources represents the external, read-only sources.
+  module Sources
+    extend self
+
+    @source_file = File.join(Dir.home, '.geturl', 'sources.yaml')
+
+    # Returns all the sources as a two dimensional Hash, with the id as key, and futher information as a child Hash.
+    #
+    # @return [Hash] The sources.
+    def get_sources
+      data = File.read(@source_file)
+      YAML.load(data).to_h rescue {}
+    end
+
+    # Registers the given sources. If the id already exists, it will be overwritten without warning.
+    #
+    # @param [String]  id     The id of the source.
+    # @param [String]  url    The url of the source.
+    # @param [Hash]  options  The options.
+    # @return [Integer]       The size of the sources.
+    def register_source(id, url, options = {})
+      id.downcase!
+      raise ArgumentError.new('The source id \'local\' is reserved.') if (id == 'local')
+      sources = get_sources || {}
+      sources[id] = {
+          'url' => url,
+          'cert' => options['cert'],
+          'key' => options['key'],
+      }
+      File.write(@source_file, sources.to_yaml)
+      return sources.size
+    end
+
+    # Unregisters the source with the given id. It also removes the locally cached file.
+    #
+    # @param [String]  id     The id of the source.
+    # @return [Integer]       The size of the sources.
+    def unregister_source(id)
+      sources = get_sources || {}
+      sources.delete(id)
+      File.write(@source_file, sources.to_yaml)
+      File.unlink(FileManager.get_local_source_file(id)) rescue nil
+      return sources.size
+    end
+
+    # Reloads the yaml file from the source, and caches it locally.
+    # If the id is nil, all sources will be reloaded.
+    #
+    # @param [Object]  id       The id of the source to reload.
+    # @param [Object]  options  The options.
+    def reload(id = nil, options = {})
+      if id.nil?
+        get_sources.keys.each {|i|
+          reload(i, options) unless i.nil?
+        }
+      else
+        begin
+          source_data = get_sources[id]
+          print "Loading #{id} ...\r" if (options[:verbose])
+
+          if source_data.include?('cert') && source_data.include?('key')
+            curl_command = "curl -s #{source_data['url']} -E #{source_data['cert']} --key #{source_data['key']}"
+            data = `#{curl_command}`
+          else
+            data = open(source_data['url']).read
+          end
+          raise Exception.new('File is empty') if data.empty?
+          file = FileManager.get_local_source_file(id)
+          FileManager.clean_and_save_items_to_yaml_file(data, file)
+          puts "Loaded #{id}" + "".ljust(5) if (options[:verbose])
+        rescue Exception => e
+          puts "Failed to load #{id}: #{e.full_message}" if (options[:verbose])
+        end
+      end
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: geturl
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Taco Jan Osinga
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mustache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+- !ruby/object:Gem::Dependency
+  name: cli-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+description: A simple command-line bookmarking tool, with support of external, read-only
+  sources.
+email: info@osingasoftware.nl
+executables:
+- geturl
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- bin/geturl
+- lib/geturl.rb
+- lib/geturl/geturl-file-manager.rb
+- lib/geturl/geturl-local-urls.rb
+- lib/geturl/geturl-searcher.rb
+- lib/geturl/geturl-sources.rb
+homepage: https://gitlab.com/osisoft-gems/geturl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: GetURL
+test_files: []