zugzwang 0.1.4 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +12 -14
- data/lib/zugzwang.rb +1 -1
- data/lib/zugzwang/cli.rb +17 -47
- data/lib/zugzwang/connection/connection.rb +14 -0
- data/lib/zugzwang/connection/populate.rb +118 -0
- data/lib/zugzwang/version.rb +1 -1
- metadata +4 -3
- data/lib/zugzwang/create.rb +0 -64
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24720d6c09dbb9b22d3f090011872c5b5c9042082877dd010e8584d79409c6bd
|
4
|
+
data.tar.gz: 2184948964ebdab0250e4ad73b4ec25be07e99b294b83f83eddc613e7b53ea3d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df3ad9c2fe7f52c4f894906e9c233197e9faeb34d81b6331b503d1c57c9f56a4c1674b39d774f12eeba57a1e29afd3fbd0c718a93e8d7e987260476421c5d1fe
|
7
|
+
data.tar.gz: 5bff1407beff6c1f405bdede301b2eaf02cec62886f3639d0c5dd0f30e1b7f2f3454c6873306506576dc650de9800f45ef706921c296c2ccb05ea01bd2defe17
|
data/README.md
CHANGED
@@ -115,7 +115,7 @@ $ zugzwang
|
|
115
115
|
The only command available is `create`.
|
116
116
|
|
117
117
|
```bash
|
118
|
-
$ zugzwang create [DATABASE] *[ITEMS] --
|
118
|
+
$ zugzwang create [DATABASE] *[ITEMS] --adapter, --adapter=ADAPTER
|
119
119
|
# Converts the metadata of PGN files from Lichess' game database to a database file.
|
120
120
|
```
|
121
121
|
|
@@ -141,44 +141,42 @@ $ zugzwang create [DATABASE] *[ITEMS] --extension, --extension=EXTENSION
|
|
141
141
|
> - `test.pgn games/*.pgn` - Does the same as the previous command, but using patterns.
|
142
142
|
> - `.` - Does the same as the previous command by recursively searching the current directory.
|
143
143
|
|
144
|
-
- `--
|
144
|
+
- `--adapter` - The database adapter to be used.
|
145
145
|
|
146
|
-
**DEFAULT**: `
|
146
|
+
**DEFAULT**: `sqlite`
|
147
147
|
|
148
|
-
The
|
149
|
-
|
150
|
-
Expected file extensions are `sql sqlite sqlite3 db`, but it is possible to override this restriction and try to generate a database with any extension, provided it is compatible with Sequel's database class constructor.
|
148
|
+
The only currently supported adapter is `sqlite`.
|
151
149
|
|
152
150
|
Ideally, the extension should be specified at the end of the command.
|
153
151
|
|
154
152
|
> **Examples**:
|
155
153
|
>
|
156
|
-
> - `--
|
157
|
-
> - `--
|
154
|
+
> - `--adapter=sqlite` - Adapter specifier with explicit equals sign.
|
155
|
+
> - `--adapter sqlite` - Adapter specifier without explicit equals sign.
|
158
156
|
|
159
157
|
### Examples
|
160
158
|
|
161
159
|
> ```bash
|
162
|
-
> $ zugzwang create lichess/db/2018-05 games-2018-05.pgn --
|
160
|
+
> $ zugzwang create lichess/db/2018-05 games-2018-05.pgn --adapter sqlite
|
163
161
|
> ```
|
164
162
|
>
|
165
163
|
> - Creates a database file at `lichess/db/2018-05.sqlite3`, populating it with data from file `games-2018-05.pgn`.
|
166
164
|
|
167
165
|
> ```bash
|
168
|
-
> $ zugzwang create database games/*.pgn --
|
166
|
+
> $ zugzwang create database games/*.pgn --adapter sqlite
|
169
167
|
> ```
|
170
168
|
>
|
171
|
-
> - Creates a database file at `database.
|
169
|
+
> - Creates a database file at `database.sqlite3`, populating it with data from `.pgn` files located within the `games` directory.
|
172
170
|
|
173
171
|
> ```bash
|
174
|
-
> $ zugzwang create games/1 .
|
172
|
+
> $ zugzwang create games/1 .
|
175
173
|
> ```
|
176
174
|
>
|
177
|
-
> - Creates a database file at `games/1.
|
175
|
+
> - Creates a database file at `games/1.sqlite3`, populating it with data from `.pgn` files found from recursively searching the current directory.
|
178
176
|
|
179
177
|
## TODO
|
180
178
|
|
181
|
-
- [ ] Add
|
179
|
+
- [ ] Add other support for other adapters (mainly Postgres)
|
182
180
|
- [ ] Add multithreaded PGN file parsing
|
183
181
|
- [ ] Add specs/testing
|
184
182
|
|
data/lib/zugzwang.rb
CHANGED
@@ -3,6 +3,7 @@ require "zugzwang/helpers"
|
|
3
3
|
require "zugzwang/cli"
|
4
4
|
|
5
5
|
module Zugzwang
|
6
|
+
ADAPTERS = %i[sqlite]
|
6
7
|
FIELDS = {
|
7
8
|
Event: {type: :string},
|
8
9
|
Site: {type: :string, size: 50},
|
@@ -22,5 +23,4 @@ module Zugzwang
|
|
22
23
|
Termination: {type: :string, size: 20},
|
23
24
|
Variant: {type: :string}
|
24
25
|
}
|
25
|
-
EXTENSIONS = %i[sql sqlite sqlite3 db]
|
26
26
|
end
|
data/lib/zugzwang/cli.rb
CHANGED
@@ -1,16 +1,14 @@
|
|
1
1
|
require 'zugzwang'
|
2
2
|
require 'zugzwang/helpers'
|
3
|
-
require 'zugzwang/
|
4
|
-
require '
|
5
|
-
require 'sqlite3'
|
6
|
-
require 'sequel'
|
3
|
+
require 'zugzwang/connection/connection'
|
4
|
+
require 'zugzwang/connection/populate'
|
7
5
|
require 'thor'
|
8
6
|
|
9
7
|
module Zugzwang
|
10
8
|
class CLI < Thor
|
11
9
|
include Thor::Actions
|
12
10
|
|
13
|
-
method_option :
|
11
|
+
method_option :adapter, type: :string, required: true, default: 'sqlite', aliases: '--adapter'
|
14
12
|
desc 'create [DATABASE] *[ITEMS]', "Converts the metadata of PGN files from Lichess' game database to a database file."
|
15
13
|
def create(database = 'lichess', *items)
|
16
14
|
if items.empty?
|
@@ -18,57 +16,29 @@ module Zugzwang
|
|
18
16
|
return
|
19
17
|
end
|
20
18
|
|
21
|
-
|
22
|
-
|
23
|
-
override = false
|
24
|
-
|
25
|
-
loop do
|
26
|
-
if Zugzwang::EXTENSIONS.include?(extension) || override
|
27
|
-
|
28
|
-
begin
|
29
|
-
db = Sequel.sqlite("#{database}.#{extension}")
|
30
|
-
Zugzwang::Create[db, database, extension, items]
|
31
|
-
rescue Sequel::DatabaseConnectionError => e
|
32
|
-
puts "\n\e[1;91mERROR\e[0m: Directory \e[1m#{File.dirname(database)}\e[0m does not exist."
|
33
|
-
|
34
|
-
pass = false
|
35
|
-
until pass
|
36
|
-
response = ask("\e[1mPROMPT: \e[0mCreate directory \e[1m#{File.dirname(database)}\e[0m? [Y/n]")
|
37
|
-
if %w[Y y YES Yes yes].include? response
|
38
|
-
puts
|
39
|
-
empty_directory(File.dirname(database))
|
40
|
-
begin
|
41
|
-
db = Sequel.sqlite("#{database}.#{extension}")
|
42
|
-
Zugzwang::Create[db, database, extension, items]
|
43
|
-
rescue Sequel::DatabaseConnectionError
|
44
|
-
puts "\n\e[1;91mERROR\e[0m: Unable to create database with extension \e[1m#{extension}\e[0m"
|
45
|
-
end
|
46
|
-
pass = true
|
47
|
-
elsif %w[N n NO No no].include? response
|
48
|
-
pass = true
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
end
|
53
|
-
|
54
|
-
return
|
55
|
-
|
56
|
-
else
|
57
|
-
|
58
|
-
puts "\n\e[1;93mWARNING\e[0m: Extension argument should be one of [#{Zugzwang::EXTENSIONS*', '}]."
|
19
|
+
database_path = sanitize_path(database)
|
20
|
+
adapter = options[:adapter].to_sym
|
59
21
|
|
22
|
+
if Zugzwang::ADAPTERS.include?(adapter)
|
23
|
+
begin
|
24
|
+
Zugzwang::Connection.new(database_path, adapter).populate(items)
|
25
|
+
rescue Sequel::DatabaseConnectionError
|
26
|
+
puts "\n\e[1;91mERROR\e[0m: Directory \e[1m#{File.dirname(database)}\e[0m does not exist."
|
60
27
|
pass = false
|
61
28
|
until pass
|
62
|
-
response = ask("\e[1mPROMPT: \e[
|
29
|
+
response = ask("\e[1mPROMPT: \e[0mCreate directory \e[1m#{File.dirname(database_path)}\e[0m? [Y/n]")
|
63
30
|
if %w[Y y YES Yes yes].include? response
|
64
|
-
|
31
|
+
puts
|
32
|
+
empty_directory(File.dirname(database_path))
|
33
|
+
Zugzwang::Connection.new(database_path,adapter).populate(items)
|
65
34
|
pass = true
|
66
35
|
elsif %w[N n NO No no].include? response
|
67
|
-
|
36
|
+
pass = true
|
68
37
|
end
|
69
38
|
end
|
70
|
-
|
71
39
|
end
|
40
|
+
else
|
41
|
+
puts "\n\e[1;91mERROR\e[0m: Database adapter argument should be one of [#{Zugzwang::ADAPTERS*', '}]."
|
72
42
|
end
|
73
43
|
end
|
74
44
|
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
module Zugzwang
|
5
|
+
class Connection
|
6
|
+
def initialize(database_path, adapter)
|
7
|
+
@database_path = database_path
|
8
|
+
@adapter = adapter
|
9
|
+
case adapter
|
10
|
+
when :sqlite then @database = Sequel.sqlite("#{database_path}.sqlite3")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'ruby-progressbar'
|
2
|
+
require 'date'
|
3
|
+
require 'time'
|
4
|
+
require 'thor'
|
5
|
+
|
6
|
+
module Zugzwang
|
7
|
+
class Connection
|
8
|
+
def populate(items)
|
9
|
+
return unless create()
|
10
|
+
|
11
|
+
files = items.map{|i|File.file?(i) ? Dir[i] : Dir["#{i}/**/*.pgn"]}.flatten
|
12
|
+
puts if files.map{|f|File.file? f}.any?{|f|f==true}
|
13
|
+
|
14
|
+
files.each do |f|
|
15
|
+
record = {}
|
16
|
+
prev = ''
|
17
|
+
|
18
|
+
# Progress bar
|
19
|
+
lines = `sed -n '=' #{f} | wc -l`.to_i
|
20
|
+
puts "\e[90mProcessing file:\e[0m \e[1m#{f}\e[0m"
|
21
|
+
progress_bar = ProgressBar.create(
|
22
|
+
:title => 'Progress',
|
23
|
+
total: lines,
|
24
|
+
progress_mark: "\e[1;35m#{?#}\e[0m",
|
25
|
+
remainder_mark: ?.,
|
26
|
+
format: "%t: %p%% (Line: %c/%C) %B"
|
27
|
+
)
|
28
|
+
|
29
|
+
# Processing file
|
30
|
+
File.open(f,'r').each do |line|
|
31
|
+
if prev =~ /\A\[.*\Z/ && line =~ /\A[\n\r].*\Z/
|
32
|
+
@database[:games].insert(**record)
|
33
|
+
record = {}
|
34
|
+
prev = line
|
35
|
+
progress_bar.increment
|
36
|
+
next
|
37
|
+
end
|
38
|
+
if line =~ /\A([\n\r]|[0-9]).*\Z/
|
39
|
+
prev = line
|
40
|
+
progress_bar.increment
|
41
|
+
next
|
42
|
+
end
|
43
|
+
subbed = line.gsub(%r{[\r\n\[\]\"]},'')
|
44
|
+
field, value = subbed.split(' ', 2)
|
45
|
+
field = field.to_sym
|
46
|
+
if FIELDS[field]
|
47
|
+
record[field] = case FIELDS[field][:type]
|
48
|
+
when :string then value
|
49
|
+
when :integer then value.to_i
|
50
|
+
when :date then Date.parse(value)
|
51
|
+
when :time then value
|
52
|
+
end
|
53
|
+
end
|
54
|
+
prev = line
|
55
|
+
progress_bar.increment
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
case @adapter
|
60
|
+
when :sqlite
|
61
|
+
puts "\n\e[92mComplete\e[0m: Populated #{@adapter} database at \e[1m#{@database_path}.sqlite3\e[0m."
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def create
|
68
|
+
begin
|
69
|
+
@database.create_table :games do
|
70
|
+
primary_key :id
|
71
|
+
FIELDS.each do |name, options|
|
72
|
+
case options[:type]
|
73
|
+
when :string then String name, size: (options[:size]||255)
|
74
|
+
when :integer then Integer name
|
75
|
+
when :date then Date name
|
76
|
+
when :time then Time name, only_time: true
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
return true
|
81
|
+
rescue Sequel::DatabaseError
|
82
|
+
pass = false
|
83
|
+
until pass
|
84
|
+
puts "\n\e[1;91mERROR\e[0m: '\e[1mgames\e[0m' table already exists in the database."
|
85
|
+
response = Ask.new.msg("\e[1mPROMPT: \e[0mOverwrite table? (\e[1mEXISTING DATA WILL BE DELETED!\e[0m) [Y/n]")
|
86
|
+
if %w[Y y YES Yes yes].include? response
|
87
|
+
@database.create_table! :games do
|
88
|
+
primary_key :id
|
89
|
+
FIELDS.each do |name, options|
|
90
|
+
case options[:type]
|
91
|
+
when :string then String name, size: (options[:size]||255)
|
92
|
+
when :integer then Integer name
|
93
|
+
when :date then Date name
|
94
|
+
when :time then Time name, only_time: true
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
pass = true
|
99
|
+
return true
|
100
|
+
elsif %w[N n NO No no].include? response
|
101
|
+
pass = true
|
102
|
+
return false
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
class Ask < Thor
|
109
|
+
include Thor::Actions
|
110
|
+
def initialize() end
|
111
|
+
|
112
|
+
no_tasks do
|
113
|
+
def msg(message)
|
114
|
+
ask(message)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
data/lib/zugzwang/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zugzwang
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edwin Onuonga
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-07-
|
11
|
+
date: 2018-07-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -124,7 +124,8 @@ files:
|
|
124
124
|
- bin/zugzwang
|
125
125
|
- lib/zugzwang.rb
|
126
126
|
- lib/zugzwang/cli.rb
|
127
|
-
- lib/zugzwang/
|
127
|
+
- lib/zugzwang/connection/connection.rb
|
128
|
+
- lib/zugzwang/connection/populate.rb
|
128
129
|
- lib/zugzwang/helpers.rb
|
129
130
|
- lib/zugzwang/version.rb
|
130
131
|
- zugzwang.gemspec
|
data/lib/zugzwang/create.rb
DELETED
@@ -1,64 +0,0 @@
|
|
1
|
-
require 'ruby-progressbar'
|
2
|
-
require 'date'
|
3
|
-
require 'time'
|
4
|
-
|
5
|
-
module Zugzwang
|
6
|
-
module Create
|
7
|
-
def self.[](database_object, database, extension, items)
|
8
|
-
database_object.create_table :games do
|
9
|
-
primary_key :id
|
10
|
-
FIELDS.each do |name, options|
|
11
|
-
case options[:type]
|
12
|
-
when :string then String name, size: (options[:size]||255)
|
13
|
-
when :integer then Integer name
|
14
|
-
when :date then Date name
|
15
|
-
when :time then Time name, only_time: true
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
files = items.map{|item|
|
21
|
-
File.file?(item) ? Dir[item] : Dir["#{item}/**/*.pgn"]
|
22
|
-
}.flatten
|
23
|
-
|
24
|
-
puts if files.map{|f|File.file? f}.any?{|f|f==true}
|
25
|
-
|
26
|
-
files.each do |f|
|
27
|
-
record = {}
|
28
|
-
prev = ''
|
29
|
-
lines = `sed -n '=' #{f} | wc -l`.to_i
|
30
|
-
puts "\e[90mProcessing file:\e[0m \e[1m#{f}\e[0m"
|
31
|
-
progress_bar = ProgressBar.create(:title => 'Progress', total: lines, progress_mark: "\e[1;35m#{?#}\e[0m", remainder_mark: ?., format: "%t: %p%% (Line: %c/%C) %B")
|
32
|
-
File.open(f,'r').each do |line|
|
33
|
-
if prev =~ /\A\[.*\Z/ && line =~ /\A[\n\r].*\Z/
|
34
|
-
database_object[:games].insert(**record)
|
35
|
-
record = {}
|
36
|
-
prev = line
|
37
|
-
progress_bar.increment
|
38
|
-
next
|
39
|
-
end
|
40
|
-
if line =~ /\A([\n\r]|[0-9]).*\Z/
|
41
|
-
prev = line
|
42
|
-
progress_bar.increment
|
43
|
-
next
|
44
|
-
end
|
45
|
-
subbed = line.gsub(%r{[\r\n\[\]\"]},'')
|
46
|
-
field, value = subbed.split(' ', 2)
|
47
|
-
field = field.to_sym
|
48
|
-
if FIELDS[field]
|
49
|
-
record[field] = case FIELDS[field][:type]
|
50
|
-
when :string then value
|
51
|
-
when :integer then value.to_i
|
52
|
-
when :date then Date.parse(value)
|
53
|
-
when :time then value
|
54
|
-
end
|
55
|
-
end
|
56
|
-
prev = line
|
57
|
-
progress_bar.increment
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
puts "\n\e[92mComplete\e[0m: Saved #{extension} database at \e[1m#{database}.#{extension}\e[0m."
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|