gigbot 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/LICENSE.txt +21 -0
- data/README.md +77 -0
- data/Rakefile +8 -0
- data/exe/gigbot +5 -0
- data/gigbot.gemspec +33 -0
- data/lib/gigbot/cli.rb +48 -0
- data/lib/gigbot/commands/deep.rb +16 -0
- data/lib/gigbot/commands/list.rb +31 -0
- data/lib/gigbot/commands/show.rb +20 -0
- data/lib/gigbot/commands/update.rb +34 -0
- data/lib/gigbot/commands.rb +4 -0
- data/lib/gigbot/formatters/base.rb +15 -0
- data/lib/gigbot/formatters/full.rb +17 -0
- data/lib/gigbot/formatters/short.rb +18 -0
- data/lib/gigbot/gig.rb +75 -0
- data/lib/gigbot/gig_writer.rb +17 -0
- data/lib/gigbot/helpers/date_helpers.rb +25 -0
- data/lib/gigbot/helpers/string_helpers.rb +14 -0
- data/lib/gigbot/parsers/base.rb +32 -0
- data/lib/gigbot/parsers/builtin.rb +41 -0
- data/lib/gigbot/parsers/flex_jobs.rb +41 -0
- data/lib/gigbot/parsers/indeed.rb +47 -0
- data/lib/gigbot/parsers/js_remotely.rb +45 -0
- data/lib/gigbot/parsers/just_remote_co.rb +50 -0
- data/lib/gigbot/parsers/remote_co.rb +50 -0
- data/lib/gigbot/parsers/rss.rb +34 -0
- data/lib/gigbot/parsers/rust_jobs.rb +35 -0
- data/lib/gigbot/parsers.rb +10 -0
- data/lib/gigbot/source.rb +66 -0
- data/lib/gigbot/sources.yml +42 -0
- data/lib/gigbot/version.rb +5 -0
- data/lib/gigbot.rb +8 -0
- data/sig/gigbot.rbs +4 -0
- metadata +78 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5d81dfe3b4dfbc0c8cdc1528c76596b3b2970e951f8db5658c3b83ddc012212c
|
4
|
+
data.tar.gz: ae0dacf5c6d3c86ce6b8a5fc8677c0e512310178ac9a4be8b982cb03f70b69f1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ed3c9ff8f4672a0a7658ee92623b6f6c967c995c67b2387cceefa455bcf8fda327638e839ee7656b3c5a1046601401f239901ba03dbf56dd3e99cb4f26693430
|
7
|
+
data.tar.gz: 937cd99ce360a51faa40899cd8d62dcaf3595b9a2dc909219ce05fa62b78e19242d836a071d180b61d654bdd12a50af864eec58a181955814a8512b2e269205c
|
data/.rspec
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2023 Teejay VanSlyke
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
# Gigbot
|
2
|
+
|
3
|
+
Git-inspired remote tech job aggregator for the command line.
|
4
|
+
|
5
|
+
**Gigbot is in active development for personal use. There will be bugs!**
|
6
|
+
|
7
|
+
When seeking a new remote software engineering gig, there are numerous job
|
8
|
+
boards all over the web. Wouldn't it be nice to be able to pull listings
|
9
|
+
from all of them and see them aggregated in one place?
|
10
|
+
|
11
|
+
Gigbot is a command line tool that scrapes job listings from various job
|
12
|
+
boards across the web and allows browsing them from the command line to
|
13
|
+
help you find your next gig!
|
14
|
+
|
15
|
+
For personal use only.
|
16
|
+
|
17
|
+
## Installation
|
18
|
+
|
19
|
+
$ gem install gigbot
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
### Job Sources
|
24
|
+
|
25
|
+
Gigbot currently pulls job listings from the following sources:
|
26
|
+
|
27
|
+
* [Indeed](https://indeed.com)
|
28
|
+
* [4 Day Week](https://4dayweek.io)
|
29
|
+
* [justremote.co](https://justremote.co)
|
30
|
+
* [Remotive](https://remotive.com)
|
31
|
+
* [remote.io](https://remote.io)
|
32
|
+
* [Rust Jobs](https://rustjobs.dev)
|
33
|
+
* [NoDesk](https://nodesk.co)
|
34
|
+
* [PyJobs](https://www.pyjobs.com)
|
35
|
+
* [JS Remotely](https://jsremotely.com)
|
36
|
+
* [We Work Remotely](https://weworkremotely.com)
|
37
|
+
* [Ruby on Remote](https://rubyonremote.com)
|
38
|
+
* [remoteok.io](https://remote.co)
|
39
|
+
|
40
|
+
### Commands
|
41
|
+
|
42
|
+
#### `gigbot update`
|
43
|
+
|
44
|
+
Fetches new jobs from all sources.
|
45
|
+
|
46
|
+
#### `gigbot list`
|
47
|
+
|
48
|
+
Lists all jobs, newest first.
|
49
|
+
|
50
|
+
#### `gigbot show <sha>`
|
51
|
+
|
52
|
+
Shows the full job listing for the job sha specificed.
|
53
|
+
|
54
|
+
#### `gigbot deep`
|
55
|
+
|
56
|
+
Deep-updates all existing jobs. This command fetches each individual job
|
57
|
+
page and extracts the full description of the job for those job boards
|
58
|
+
that lack RSS feeds.
|
59
|
+
|
60
|
+
#### `gigbot clean`
|
61
|
+
|
62
|
+
Purges all job data.
|
63
|
+
|
64
|
+
## Development
|
65
|
+
|
66
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
67
|
+
|
68
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
69
|
+
|
70
|
+
## Contributing
|
71
|
+
|
72
|
+
Bug reports and pull requests are welcome on GitHub at
|
73
|
+
https://github.com/teejayvanslyke/gigbot.
|
74
|
+
|
75
|
+
## License
|
76
|
+
|
77
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/exe/gigbot
ADDED
data/gigbot.gemspec
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/gigbot/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "gigbot"
|
7
|
+
spec.version = Gigbot::VERSION
|
8
|
+
spec.authors = ["Teejay VanSlyke"]
|
9
|
+
spec.email = ["root@teejayvanslyke.com"]
|
10
|
+
|
11
|
+
spec.summary = "Git-inspired remote tech job aggregator for the command line"
|
12
|
+
spec.homepage = "https://github.com/teejayvanslyke/gigbot"
|
13
|
+
spec.license = "MIT"
|
14
|
+
spec.required_ruby_version = ">= 2.6.0"
|
15
|
+
|
16
|
+
# Specify which files should be added to the gem when it is released.
|
17
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
18
|
+
spec.files = Dir.chdir(__dir__) do
|
19
|
+
`git ls-files -z`.split("\x0").reject do |f|
|
20
|
+
(File.expand_path(f) == __FILE__) ||
|
21
|
+
f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
|
22
|
+
end
|
23
|
+
end
|
24
|
+
spec.bindir = "exe"
|
25
|
+
spec.executables << "gigbot"
|
26
|
+
spec.require_paths = ["lib"]
|
27
|
+
|
28
|
+
# Uncomment to register a new dependency of your gem
|
29
|
+
# spec.add_dependency "example-gem", "~> 1.0"
|
30
|
+
|
31
|
+
# For more information and examples about making a new gem, check out our
|
32
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
33
|
+
end
|
data/lib/gigbot/cli.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'thor'
|
3
|
+
require 'date'
|
4
|
+
|
5
|
+
require_relative 'commands'
|
6
|
+
|
7
|
+
module Gigbot
|
8
|
+
class CLI < Thor
|
9
|
+
desc "update", "Updates jobs from all sources"
|
10
|
+
method_option :verbose, type: :boolean, aliases: 'v'
|
11
|
+
def update
|
12
|
+
Gigbot::Commands::Update.run(options)
|
13
|
+
end
|
14
|
+
|
15
|
+
desc "list", "Lists jobs from all sources"
|
16
|
+
def list
|
17
|
+
Gigbot::Commands::List.run
|
18
|
+
end
|
19
|
+
|
20
|
+
desc "today", "Lists jobs for past 24 hours"
|
21
|
+
def today
|
22
|
+
Gigbot::Commands::List.run(since: Time.now - (60 * 60 * 24))
|
23
|
+
end
|
24
|
+
|
25
|
+
desc "clean", "Clears data"
|
26
|
+
def clean
|
27
|
+
Gigbot::Gig.clean!
|
28
|
+
end
|
29
|
+
|
30
|
+
desc "deep", "Fetches detailed metadata for all existing jobs"
|
31
|
+
def deep
|
32
|
+
Gigbot::Commands::Deep.run
|
33
|
+
end
|
34
|
+
|
35
|
+
desc "search", "Searches jobs by keyword"
|
36
|
+
def search(query)
|
37
|
+
Gigbot::Commands::List.run(query: query)
|
38
|
+
end
|
39
|
+
|
40
|
+
desc "show", "Show a full job listing"
|
41
|
+
def show(sha)
|
42
|
+
Gigbot::Commands::Show.run(sha)
|
43
|
+
end
|
44
|
+
|
45
|
+
map "up" => "update"
|
46
|
+
map "ls" => "list"
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
require 'tty-pager'
|
3
|
+
|
4
|
+
require_relative '../gig'
|
5
|
+
require_relative '../gig_writer'
|
6
|
+
|
7
|
+
module Gigbot
|
8
|
+
module Commands
|
9
|
+
class List
|
10
|
+
def self.run(options = {})
|
11
|
+
new.run(options)
|
12
|
+
end
|
13
|
+
|
14
|
+
def run(options = {})
|
15
|
+
formatter = options[:formatter] || Gigbot::Formatters::Short
|
16
|
+
TTY::Pager.page do |pager|
|
17
|
+
writer = GigWriter.new(pager, formatter)
|
18
|
+
if options[:since]
|
19
|
+
Gig.since(options[:since]).each do |gig|
|
20
|
+
writer.write(gig)
|
21
|
+
end
|
22
|
+
else
|
23
|
+
Gig.all.each do |gig|
|
24
|
+
writer.write(gig)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require_relative '../gig_writer'
|
2
|
+
require_relative '../formatters/full'
|
3
|
+
|
4
|
+
module Gigbot
|
5
|
+
module Commands
|
6
|
+
class Show
|
7
|
+
def self.run(sha)
|
8
|
+
new.run(sha)
|
9
|
+
end
|
10
|
+
|
11
|
+
def run(sha)
|
12
|
+
TTY::Pager.page do |pager|
|
13
|
+
writer = GigWriter.new(pager, Gigbot::Formatters::Full)
|
14
|
+
gig = Gig.find(sha)
|
15
|
+
writer.write(gig)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require_relative '../source'
|
2
|
+
|
3
|
+
module Gigbot
|
4
|
+
module Commands
|
5
|
+
class Update
|
6
|
+
def self.run(options={})
|
7
|
+
new.run(options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def run(options={})
|
11
|
+
Gigbot::Source.each do |source|
|
12
|
+
begin
|
13
|
+
source.import
|
14
|
+
puts [
|
15
|
+
"✓".colorize(color: :green),
|
16
|
+
"(#{source.imported.length})".ljust(7, ' ').colorize(color: :yellow),
|
17
|
+
source.title,
|
18
|
+
].join(' ')
|
19
|
+
rescue StandardError => e
|
20
|
+
if options[:verbose]
|
21
|
+
puts e
|
22
|
+
puts e.backtrace
|
23
|
+
end
|
24
|
+
puts [
|
25
|
+
"X".colorize(color: :red),
|
26
|
+
"".ljust(7, ' '),
|
27
|
+
source.title,
|
28
|
+
].join(' ')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require_relative './base'
|
2
|
+
|
3
|
+
module Gigbot
|
4
|
+
module Formatters
|
5
|
+
class Short < Base
|
6
|
+
def to_s
|
7
|
+
[
|
8
|
+
"job #{gig.id}".colorize(color: :yellow),
|
9
|
+
"Title: #{gig.title}",
|
10
|
+
"URL: #{gig.url}",
|
11
|
+
"Source: #{gig.source.title}",
|
12
|
+
"Date: #{gig.created_at}"
|
13
|
+
].join("\n")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
data/lib/gigbot/gig.rb
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'digest'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module Gigbot
|
5
|
+
class Gig
|
6
|
+
DATA_PATH = File.dirname(__FILE__) + '/../../data'
|
7
|
+
|
8
|
+
def initialize(attributes = {})
|
9
|
+
@title = attributes[:title]
|
10
|
+
@url = attributes[:url]
|
11
|
+
@created_at = attributes[:created_at]
|
12
|
+
@source = Source.find(attributes[:source_id])
|
13
|
+
@description = attributes[:description]
|
14
|
+
end
|
15
|
+
|
16
|
+
attr_accessor :title, :url, :created_at, :source, :description
|
17
|
+
|
18
|
+
def as_json
|
19
|
+
{
|
20
|
+
id: id,
|
21
|
+
title: title,
|
22
|
+
url: url,
|
23
|
+
created_at: created_at,
|
24
|
+
source_id: source.id,
|
25
|
+
description: description,
|
26
|
+
}
|
27
|
+
end
|
28
|
+
|
29
|
+
def id
|
30
|
+
Digest::SHA1.hexdigest(url)
|
31
|
+
end
|
32
|
+
|
33
|
+
def save
|
34
|
+
File.open(DATA_PATH + '/' + id + '.yml', 'w') {|f| f.write(YAML.dump(as_json))}
|
35
|
+
true
|
36
|
+
end
|
37
|
+
|
38
|
+
def update_attributes(attributes)
|
39
|
+
attributes.each do |key, value|
|
40
|
+
instance_variable_set("@#{key}", value)
|
41
|
+
end
|
42
|
+
|
43
|
+
save
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_s
|
47
|
+
title
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.clean!
|
51
|
+
FileUtils.rm_rf Dir.glob(File.join(DATA_PATH, '*.yml'))
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.all
|
55
|
+
Dir[DATA_PATH + '/*.yml'].map {|path| from_yaml(path)}.sort_by(&:created_at).reverse
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.each
|
59
|
+
all.each {|gig| yield gig}
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.find(id)
|
63
|
+
from_yaml(DATA_PATH + '/' + id + '.yml')
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.since(date)
|
67
|
+
all.select {|gig| gig.created_at > date}
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.from_yaml(yaml_path)
|
71
|
+
data = YAML.load_file(yaml_path)
|
72
|
+
new(data)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require_relative './formatters/short'
|
2
|
+
|
3
|
+
module Gigbot
|
4
|
+
class GigWriter
|
5
|
+
def initialize(io, formatter_class)
|
6
|
+
@io = io
|
7
|
+
@formatter_class = formatter_class
|
8
|
+
end
|
9
|
+
|
10
|
+
attr_reader :io, :formatter_class
|
11
|
+
|
12
|
+
def write(gig)
|
13
|
+
io.puts formatter_class.new(gig).to_s
|
14
|
+
io.puts ""
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Gigbot
|
2
|
+
module Helpers
|
3
|
+
module DateHelpers
|
4
|
+
def dehumanize_date(humanized_date)
|
5
|
+
parts = humanized_date.downcase.split(' ')
|
6
|
+
return Time.now if parts[0] == 'today'
|
7
|
+
quantity = if ['a', 'an'].include?(parts[0].strip) then 1 else parts[0].to_i end
|
8
|
+
unit = parts[1]
|
9
|
+
|
10
|
+
seconds = case unit
|
11
|
+
when /hour/
|
12
|
+
60 * 60
|
13
|
+
when /day/
|
14
|
+
60 * 60 * 24
|
15
|
+
when /week/
|
16
|
+
60 * 60 * 24 * 7
|
17
|
+
when /month/
|
18
|
+
60 * 60 * 24 * 30
|
19
|
+
end
|
20
|
+
|
21
|
+
Time.now - (quantity * seconds)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Gigbot
|
2
|
+
module Parsers
|
3
|
+
@registry = {}
|
4
|
+
|
5
|
+
def self.register(name, klass)
|
6
|
+
@registry[name] = klass
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.[](name)
|
10
|
+
@registry[name]
|
11
|
+
end
|
12
|
+
|
13
|
+
class Base
|
14
|
+
def initialize(url)
|
15
|
+
@url = url
|
16
|
+
end
|
17
|
+
|
18
|
+
attr_reader :url
|
19
|
+
|
20
|
+
def title
|
21
|
+
self.class.name
|
22
|
+
end
|
23
|
+
|
24
|
+
def parse
|
25
|
+
end
|
26
|
+
|
27
|
+
def parse_deep(gig)
|
28
|
+
{}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'nokogiri'
|
4
|
+
require_relative '../helpers/date_helpers'
|
5
|
+
|
6
|
+
module Gigbot
|
7
|
+
module Parsers
|
8
|
+
class Builtin < Base
|
9
|
+
include Gigbot::Helpers::DateHelpers
|
10
|
+
|
11
|
+
def title
|
12
|
+
"BuiltIn"
|
13
|
+
end
|
14
|
+
|
15
|
+
def parse
|
16
|
+
URI.open(url) do |file|
|
17
|
+
doc = Nokogiri::HTML(file)
|
18
|
+
doc.css('[data-id="job-card"]').each do |card|
|
19
|
+
yield parse_card(card)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def parse_card(card)
|
25
|
+
title = card.css('h2 a').text.strip
|
26
|
+
url = "https://www.builtin.com" + card.css('h2 a').attribute('href').value
|
27
|
+
id = url
|
28
|
+
date_css = 'div#main.row div.col-12.col-lg-6.bounded-attribute-section.d-flex.align-items-start.align-items-lg-center.fs-md.flex-column.flex-lg-row div.d-flex.flex-grow-1.gap-lg div.d-flex.flex-column.gap-0.flex-md-row.gap-md-md.flex-lg-column.gap-lg-0.fill-even div.d-flex.align-items-start.gap-sm span.font-barlow.text-gray-03'
|
29
|
+
created_at = dehumanize_date(card.css(date_css).first.text.strip)
|
30
|
+
|
31
|
+
{
|
32
|
+
title: title,
|
33
|
+
url: url,
|
34
|
+
created_at: created_at
|
35
|
+
}
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
Gigbot::Parsers.register('builtin', Gigbot::Parsers::Builtin)
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'nokogiri'
|
4
|
+
require_relative '../helpers/date_helpers'
|
5
|
+
|
6
|
+
module Gigbot
|
7
|
+
module Parsers
|
8
|
+
class FlexJobs < Base
|
9
|
+
include Gigbot::Helpers::DateHelpers
|
10
|
+
|
11
|
+
def title
|
12
|
+
"FlexJobs"
|
13
|
+
end
|
14
|
+
|
15
|
+
def parse
|
16
|
+
URI.open(url) do |file|
|
17
|
+
doc = Nokogiri::HTML(file)
|
18
|
+
doc.css('li.job').each do |card|
|
19
|
+
yield parse_card(card)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def parse_card(card)
|
25
|
+
title = card.attribute('data-title').value
|
26
|
+
url = 'https://www.flexjobs.com' + card.attribute('data-url').value
|
27
|
+
id = url
|
28
|
+
date = card.css('.job-age').first.text.gsub('New!', '').strip
|
29
|
+
created_at = dehumanize_date(date)
|
30
|
+
|
31
|
+
{
|
32
|
+
title: title,
|
33
|
+
url: url,
|
34
|
+
created_at: created_at,
|
35
|
+
}
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
Gigbot::Parsers.register('flexjobs', Gigbot::Parsers::FlexJobs)
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'ferrum'
|
4
|
+
require_relative '../helpers/date_helpers'
|
5
|
+
require_relative '../helpers/string_helpers'
|
6
|
+
|
7
|
+
module Gigbot
|
8
|
+
module Parsers
|
9
|
+
class Indeed < Base
|
10
|
+
include Gigbot::Helpers::DateHelpers
|
11
|
+
include Gigbot::Helpers::StringHelpers
|
12
|
+
|
13
|
+
def title
|
14
|
+
"Indeed"
|
15
|
+
end
|
16
|
+
|
17
|
+
def parse
|
18
|
+
browser = Ferrum::Browser.new
|
19
|
+
browser.headers.set({
|
20
|
+
"User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0"
|
21
|
+
})
|
22
|
+
browser.goto(url)
|
23
|
+
browser.network.wait_for_idle
|
24
|
+
browser.css(".job_seen_beacon").each do |item|
|
25
|
+
link = item.at_css('.jcs-JobTitle')
|
26
|
+
title = link.at_css('span').text
|
27
|
+
date = item.
|
28
|
+
at_css(".underShelfFooter span.date").
|
29
|
+
text.
|
30
|
+
gsub('Posted', '').
|
31
|
+
gsub('Active', '').
|
32
|
+
gsub('Employer', '')
|
33
|
+
created_at = dehumanize_date(date)
|
34
|
+
url = "https://www.indeed.com" + link.attribute('href')
|
35
|
+
|
36
|
+
yield({
|
37
|
+
title: title,
|
38
|
+
url: url,
|
39
|
+
created_at: created_at,
|
40
|
+
})
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
Gigbot::Parsers.register('indeed', Gigbot::Parsers::Indeed)
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require_relative '../helpers/date_helpers'
|
2
|
+
require_relative '../helpers/string_helpers'
|
3
|
+
|
4
|
+
module Gigbot
|
5
|
+
module Parsers
|
6
|
+
class JsRemotely < Base
|
7
|
+
include Gigbot::Helpers::DateHelpers
|
8
|
+
include Gigbot::Helpers::StringHelpers
|
9
|
+
|
10
|
+
def title
|
11
|
+
"JS Remotely"
|
12
|
+
end
|
13
|
+
|
14
|
+
def parse
|
15
|
+
URI.open(url) do |file|
|
16
|
+
doc = Nokogiri::HTML(file)
|
17
|
+
doc.css('[data-cy="jobList"] > div').each do |entry|
|
18
|
+
link = entry.css('[data-cy="jobTitle"] > a').first
|
19
|
+
next unless link
|
20
|
+
|
21
|
+
title = link.text
|
22
|
+
url = "https://jsremotely.com" + link.attribute('href').value
|
23
|
+
created_at = dehumanize_date(entry.css('div:last-child p').last.text)
|
24
|
+
|
25
|
+
yield({
|
26
|
+
title: title,
|
27
|
+
url: url,
|
28
|
+
created_at: created_at
|
29
|
+
})
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def parse_deep(gig)
|
35
|
+
URI.open(gig.url) do |file|
|
36
|
+
doc = Nokogiri::HTML(file)
|
37
|
+
description = textify_html_summary(doc.xpath("//div[starts-with(@class, 'JobBody_jobBodyText')]").first.inner_html)
|
38
|
+
return { description: description }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
Gigbot::Parsers.register('js-remotely', Gigbot::Parsers::JsRemotely)
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'nokogiri'
|
4
|
+
require_relative '../helpers/date_helpers'
|
5
|
+
require_relative '../helpers/string_helpers'
|
6
|
+
|
7
|
+
module Gigbot
|
8
|
+
module Parsers
|
9
|
+
class JustRemoteCo < Base
|
10
|
+
include Gigbot::Helpers::DateHelpers
|
11
|
+
include Gigbot::Helpers::StringHelpers
|
12
|
+
|
13
|
+
def title
|
14
|
+
"JustRemote.co"
|
15
|
+
end
|
16
|
+
|
17
|
+
def parse_date(date_str)
|
18
|
+
Date.parse(date_str).to_time
|
19
|
+
end
|
20
|
+
|
21
|
+
def parse
|
22
|
+
browser = Ferrum::Browser.new
|
23
|
+
browser.goto(url)
|
24
|
+
browser.network.wait_for_idle
|
25
|
+
browser.xpath("//div[starts-with(@class, 'new-job-item__jobitemwrapper')]").each do |item|
|
26
|
+
link = item.at_xpath("//a[starts-with(@class, 'new-job-item__JobMeta')]")
|
27
|
+
created_at = parse_date(item.at_xpath("//div[starts-with(@class, 'new-job-item__JobItemDate')]").text)
|
28
|
+
title = item.at_css('h3').text
|
29
|
+
url = "https://justremote.co/" + link.attribute('href')
|
30
|
+
|
31
|
+
yield({
|
32
|
+
title: title,
|
33
|
+
url: url,
|
34
|
+
created_at: created_at,
|
35
|
+
})
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def parse_deep(gig)
|
40
|
+
URI.open(gig.url) do |file|
|
41
|
+
doc = Nokogiri::HTML(file)
|
42
|
+
description = textify_html_summary(doc.xpath("//div[starts-with(@class, 'job-textarea__Content')]").first.inner_html)
|
43
|
+
return { description: description }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
Gigbot::Parsers.register('justremote.co', Gigbot::Parsers::JustRemoteCo)
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'nokogiri'
|
4
|
+
require_relative '../helpers/date_helpers'
|
5
|
+
require_relative '../helpers/string_helpers'
|
6
|
+
|
7
|
+
module Gigbot
|
8
|
+
module Parsers
|
9
|
+
class RemoteCo < Base
|
10
|
+
include Gigbot::Helpers::DateHelpers
|
11
|
+
include Gigbot::Helpers::StringHelpers
|
12
|
+
|
13
|
+
def title
|
14
|
+
"Remote.co"
|
15
|
+
end
|
16
|
+
|
17
|
+
def parse
|
18
|
+
URI.open(url) do |file|
|
19
|
+
doc = Nokogiri::HTML(file)
|
20
|
+
doc.css('.card-body a.card.m-0').each do |card|
|
21
|
+
yield parse_card(card)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def parse_card(card)
|
27
|
+
title = card.css('p.m-0 .font-weight-bold').text.strip
|
28
|
+
url = 'https://remote.co' + card.attribute('href').value.strip
|
29
|
+
id = url
|
30
|
+
created_at = dehumanize_date(card.css('date').first.text.strip)
|
31
|
+
|
32
|
+
{
|
33
|
+
title: title,
|
34
|
+
url: url,
|
35
|
+
created_at: created_at,
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
def parse_deep(gig)
|
40
|
+
URI.open(gig.url) do |file|
|
41
|
+
doc = Nokogiri::HTML(file)
|
42
|
+
description = textify_html_summary(doc.css(".job_description").first.inner_html)
|
43
|
+
return { description: description }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
Gigbot::Parsers.register('remote.co', Gigbot::Parsers::RemoteCo)
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'feedjira'
|
2
|
+
require 'open-uri'
|
3
|
+
|
4
|
+
module Gigbot
|
5
|
+
module Parsers
|
6
|
+
class RSS < Base
|
7
|
+
include Gigbot::Helpers::StringHelpers
|
8
|
+
|
9
|
+
def open_uri(&block)
|
10
|
+
URI.open(url, read_timeout: 10, &block)
|
11
|
+
end
|
12
|
+
|
13
|
+
def title
|
14
|
+
@title ||= Feedjira.parse(open_uri.read).title
|
15
|
+
end
|
16
|
+
|
17
|
+
def parse
|
18
|
+
open_uri do |rss|
|
19
|
+
feed = Feedjira.parse(rss.read)
|
20
|
+
feed.entries.each do |rss_item|
|
21
|
+
yield({
|
22
|
+
title: rss_item.title.strip,
|
23
|
+
url: rss_item.url.strip,
|
24
|
+
description: textify_html_summary(rss_item.summary&.strip),
|
25
|
+
created_at: rss_item.published,
|
26
|
+
})
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
Gigbot::Parsers.register('rss', Gigbot::Parsers::RSS)
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'ferrum'
|
3
|
+
|
4
|
+
module Gigbot
|
5
|
+
module Parsers
|
6
|
+
class RustJobs < Base
|
7
|
+
include Gigbot::Helpers::DateHelpers
|
8
|
+
|
9
|
+
def title
|
10
|
+
"Rust Jobs"
|
11
|
+
end
|
12
|
+
|
13
|
+
def parse
|
14
|
+
browser = Ferrum::Browser.new
|
15
|
+
browser.goto(url)
|
16
|
+
browser.network.wait_for_idle
|
17
|
+
results = browser.at_css('.max-w-7xl > .my-5 ul')
|
18
|
+
results.css('li').each do |entry|
|
19
|
+
link = entry.at_css('h2 > a')
|
20
|
+
next unless link
|
21
|
+
title = link.text.strip
|
22
|
+
url = "https://rustjobs.dev" + link.attribute('href')
|
23
|
+
created_at = Time.parse(entry.at_css('time').attribute('datetime'))
|
24
|
+
yield({
|
25
|
+
title: title,
|
26
|
+
url: url,
|
27
|
+
created_at: created_at,
|
28
|
+
})
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
Gigbot::Parsers.register('rust-jobs', Gigbot::Parsers::RustJobs)
|
@@ -0,0 +1,10 @@
|
|
1
|
+
require_relative './parsers/base'
|
2
|
+
require_relative './parsers/remote_co'
|
3
|
+
require_relative './parsers/flex_jobs'
|
4
|
+
require_relative './parsers/js_remotely'
|
5
|
+
require_relative './parsers/rust_jobs'
|
6
|
+
require_relative './parsers/just_remote_co'
|
7
|
+
require_relative './parsers/builtin'
|
8
|
+
require_relative './parsers/indeed'
|
9
|
+
require_relative './parsers/rss'
|
10
|
+
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'open-uri'
|
3
|
+
require_relative './gig'
|
4
|
+
require_relative './parsers'
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
module Gigbot
|
8
|
+
class Source
|
9
|
+
SOURCES_YAML = File.dirname(__FILE__) + '/sources.yml'
|
10
|
+
attr_reader :url, :parser_name, :title, :imported
|
11
|
+
|
12
|
+
def initialize(url, parser_name, title)
|
13
|
+
@url = url
|
14
|
+
@parser_name = parser_name
|
15
|
+
@title = title
|
16
|
+
@imported = []
|
17
|
+
end
|
18
|
+
|
19
|
+
def parser_class
|
20
|
+
Gigbot::Parsers[parser_name]
|
21
|
+
end
|
22
|
+
|
23
|
+
def parser
|
24
|
+
@parser ||= parser_class.new(url)
|
25
|
+
end
|
26
|
+
|
27
|
+
def id
|
28
|
+
Digest::SHA1.hexdigest(url)
|
29
|
+
end
|
30
|
+
|
31
|
+
def import
|
32
|
+
@imported = []
|
33
|
+
parser.parse do |params|
|
34
|
+
gig = Gig.new(params.merge(source_id: self.id))
|
35
|
+
gig.save
|
36
|
+
@imported << gig
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def import_deep(gig)
|
41
|
+
params = parser.parse_deep(gig)
|
42
|
+
if params.length > 0
|
43
|
+
return gig.update_attributes(params)
|
44
|
+
else
|
45
|
+
return false
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.all
|
50
|
+
definitions = YAML.load_file(SOURCES_YAML)
|
51
|
+
definitions.map do |definition|
|
52
|
+
new(definition['url'], definition['parser'], definition['title'])
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.each
|
57
|
+
all.each do |source|
|
58
|
+
yield source
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.find(id)
|
63
|
+
all.find {|source| source.id == id}
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
- url: https://www.indeed.com/jobs?q=web+engineer&l=Remote
|
2
|
+
parser: indeed
|
3
|
+
title: Indeed
|
4
|
+
- url: https://4dayweek.io/rss
|
5
|
+
parser: rss
|
6
|
+
title: 4 Day Week
|
7
|
+
- url: https://justremote.co/remote-developer-jobs
|
8
|
+
parser: justremote.co
|
9
|
+
title: justremote.co
|
10
|
+
- url: https://remotive.com/remote-jobs/feed/software-dev
|
11
|
+
parser: rss
|
12
|
+
title: Remotive
|
13
|
+
- url: https://s3.remote.io/feed/rss.xml
|
14
|
+
parser: rss
|
15
|
+
title: remote.io
|
16
|
+
- url: https://rustjobs.dev/
|
17
|
+
parser: rust-jobs
|
18
|
+
title: Rust Jobs
|
19
|
+
- url: https://nodesk.co/remote-jobs/engineering/index.xml
|
20
|
+
parser: rss
|
21
|
+
title: NoDesk
|
22
|
+
- url: https://www.skipthedrive.com/search/Web+Development/feed/rss2/
|
23
|
+
parser: rss
|
24
|
+
title: Skip The Drive
|
25
|
+
- url: https://www.pyjobs.com/rss
|
26
|
+
parser: rss
|
27
|
+
title: PyJobs
|
28
|
+
- url: https://jsremotely.com/
|
29
|
+
parser: js-remotely
|
30
|
+
title: JS Remotely
|
31
|
+
- url: https://weworkremotely.com/remote-jobs.rss
|
32
|
+
parser: rss
|
33
|
+
title: We Work Remotely
|
34
|
+
- url: https://rubyonremote.com/jobs.rss
|
35
|
+
parser: rss
|
36
|
+
title: Ruby on Remote
|
37
|
+
- url: https://remoteok.io/remote-jobs.rss
|
38
|
+
parser: rss
|
39
|
+
title: remoteok.io
|
40
|
+
- url: https://remote.co/remote-jobs/developer/
|
41
|
+
parser: remote.co
|
42
|
+
title: remote.co
|
data/lib/gigbot.rb
ADDED
data/sig/gigbot.rbs
ADDED
metadata
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gigbot
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Teejay VanSlyke
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-01-10 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description:
|
14
|
+
email:
|
15
|
+
- root@teejayvanslyke.com
|
16
|
+
executables:
|
17
|
+
- gigbot
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- ".rspec"
|
22
|
+
- LICENSE.txt
|
23
|
+
- README.md
|
24
|
+
- Rakefile
|
25
|
+
- exe/gigbot
|
26
|
+
- gigbot.gemspec
|
27
|
+
- lib/gigbot.rb
|
28
|
+
- lib/gigbot/cli.rb
|
29
|
+
- lib/gigbot/commands.rb
|
30
|
+
- lib/gigbot/commands/deep.rb
|
31
|
+
- lib/gigbot/commands/list.rb
|
32
|
+
- lib/gigbot/commands/show.rb
|
33
|
+
- lib/gigbot/commands/update.rb
|
34
|
+
- lib/gigbot/formatters/base.rb
|
35
|
+
- lib/gigbot/formatters/full.rb
|
36
|
+
- lib/gigbot/formatters/short.rb
|
37
|
+
- lib/gigbot/gig.rb
|
38
|
+
- lib/gigbot/gig_writer.rb
|
39
|
+
- lib/gigbot/helpers/date_helpers.rb
|
40
|
+
- lib/gigbot/helpers/string_helpers.rb
|
41
|
+
- lib/gigbot/parsers.rb
|
42
|
+
- lib/gigbot/parsers/base.rb
|
43
|
+
- lib/gigbot/parsers/builtin.rb
|
44
|
+
- lib/gigbot/parsers/flex_jobs.rb
|
45
|
+
- lib/gigbot/parsers/indeed.rb
|
46
|
+
- lib/gigbot/parsers/js_remotely.rb
|
47
|
+
- lib/gigbot/parsers/just_remote_co.rb
|
48
|
+
- lib/gigbot/parsers/remote_co.rb
|
49
|
+
- lib/gigbot/parsers/rss.rb
|
50
|
+
- lib/gigbot/parsers/rust_jobs.rb
|
51
|
+
- lib/gigbot/source.rb
|
52
|
+
- lib/gigbot/sources.yml
|
53
|
+
- lib/gigbot/version.rb
|
54
|
+
- sig/gigbot.rbs
|
55
|
+
homepage: https://github.com/teejayvanslyke/gigbot
|
56
|
+
licenses:
|
57
|
+
- MIT
|
58
|
+
metadata: {}
|
59
|
+
post_install_message:
|
60
|
+
rdoc_options: []
|
61
|
+
require_paths:
|
62
|
+
- lib
|
63
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: 2.6.0
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '0'
|
73
|
+
requirements: []
|
74
|
+
rubygems_version: 3.2.3
|
75
|
+
signing_key:
|
76
|
+
specification_version: 4
|
77
|
+
summary: Git-inspired remote tech job aggregator for the command line
|
78
|
+
test_files: []
|