zugzwang 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -14
- data/lib/zugzwang.rb +1 -1
- data/lib/zugzwang/cli.rb +17 -47
- data/lib/zugzwang/connection/connection.rb +14 -0
- data/lib/zugzwang/connection/populate.rb +118 -0
- data/lib/zugzwang/version.rb +1 -1
- metadata +4 -3
- data/lib/zugzwang/create.rb +0 -64
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24720d6c09dbb9b22d3f090011872c5b5c9042082877dd010e8584d79409c6bd
|
4
|
+
data.tar.gz: 2184948964ebdab0250e4ad73b4ec25be07e99b294b83f83eddc613e7b53ea3d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df3ad9c2fe7f52c4f894906e9c233197e9faeb34d81b6331b503d1c57c9f56a4c1674b39d774f12eeba57a1e29afd3fbd0c718a93e8d7e987260476421c5d1fe
|
7
|
+
data.tar.gz: 5bff1407beff6c1f405bdede301b2eaf02cec62886f3639d0c5dd0f30e1b7f2f3454c6873306506576dc650de9800f45ef706921c296c2ccb05ea01bd2defe17
|
data/README.md
CHANGED
@@ -115,7 +115,7 @@ $ zugzwang
|
|
115
115
|
The only command available is `create`.
|
116
116
|
|
117
117
|
```bash
|
118
|
-
$ zugzwang create [DATABASE] *[ITEMS] --
|
118
|
+
$ zugzwang create [DATABASE] *[ITEMS] --adapter, --adapter=ADAPTER
|
119
119
|
# Converts the metadata of PGN files from Lichess' game database to a database file.
|
120
120
|
```
|
121
121
|
|
@@ -141,44 +141,42 @@ $ zugzwang create [DATABASE] *[ITEMS] --extension, --extension=EXTENSION
|
|
141
141
|
> - `test.pgn games/*.pgn` - Does the same as the previous command, but using patterns.
|
142
142
|
> - `.` - Does the same as the previous command by recursively searching the current directory.
|
143
143
|
|
144
|
-
- `--
|
144
|
+
- `--adapter` - The database adapter to be used.
|
145
145
|
|
146
|
-
**DEFAULT**: `
|
146
|
+
**DEFAULT**: `sqlite`
|
147
147
|
|
148
|
-
The
|
149
|
-
|
150
|
-
Expected file extensions are `sql sqlite sqlite3 db`, but it is possible to override this restriction and try to generate a database with any extension, provided it is compatible with Sequel's database class constructor.
|
148
|
+
The only currently supported adapter is `sqlite`.
|
151
149
|
|
152
150
|
Ideally, the extension should be specified at the end of the command.
|
153
151
|
|
154
152
|
> **Examples**:
|
155
153
|
>
|
156
|
-
> - `--
|
157
|
-
> - `--
|
154
|
+
> - `--adapter=sqlite` - Adapter specifier with explicit equals sign.
|
155
|
+
> - `--adapter sqlite` - Adapter specifier without explicit equals sign.
|
158
156
|
|
159
157
|
### Examples
|
160
158
|
|
161
159
|
> ```bash
|
162
|
-
> $ zugzwang create lichess/db/2018-05 games-2018-05.pgn --
|
160
|
+
> $ zugzwang create lichess/db/2018-05 games-2018-05.pgn --adapter sqlite
|
163
161
|
> ```
|
164
162
|
>
|
165
163
|
> - Creates a database file at `lichess/db/2018-05.sqlite3`, populating it with data from file `games-2018-05.pgn`.
|
166
164
|
|
167
165
|
> ```bash
|
168
|
-
> $ zugzwang create database games/*.pgn --
|
166
|
+
> $ zugzwang create database games/*.pgn --adapter sqlite
|
169
167
|
> ```
|
170
168
|
>
|
171
|
-
> - Creates a database file at `database.
|
169
|
+
> - Creates a database file at `database.sqlite3`, populating it with data from `.pgn` files located within the `games` directory.
|
172
170
|
|
173
171
|
> ```bash
|
174
|
-
> $ zugzwang create games/1 .
|
172
|
+
> $ zugzwang create games/1 .
|
175
173
|
> ```
|
176
174
|
>
|
177
|
-
> - Creates a database file at `games/1.
|
175
|
+
> - Creates a database file at `games/1.sqlite3`, populating it with data from `.pgn` files found from recursively searching the current directory.
|
178
176
|
|
179
177
|
## TODO
|
180
178
|
|
181
|
-
- [ ] Add
|
179
|
+
- [ ] Add other support for other adapters (mainly Postgres)
|
182
180
|
- [ ] Add multithreaded PGN file parsing
|
183
181
|
- [ ] Add specs/testing
|
184
182
|
|
data/lib/zugzwang.rb
CHANGED
@@ -3,6 +3,7 @@ require "zugzwang/helpers"
|
|
3
3
|
require "zugzwang/cli"
|
4
4
|
|
5
5
|
module Zugzwang
|
6
|
+
ADAPTERS = %i[sqlite]
|
6
7
|
FIELDS = {
|
7
8
|
Event: {type: :string},
|
8
9
|
Site: {type: :string, size: 50},
|
@@ -22,5 +23,4 @@ module Zugzwang
|
|
22
23
|
Termination: {type: :string, size: 20},
|
23
24
|
Variant: {type: :string}
|
24
25
|
}
|
25
|
-
EXTENSIONS = %i[sql sqlite sqlite3 db]
|
26
26
|
end
|
data/lib/zugzwang/cli.rb
CHANGED
@@ -1,16 +1,14 @@
|
|
1
1
|
require 'zugzwang'
|
2
2
|
require 'zugzwang/helpers'
|
3
|
-
require 'zugzwang/
|
4
|
-
require '
|
5
|
-
require 'sqlite3'
|
6
|
-
require 'sequel'
|
3
|
+
require 'zugzwang/connection/connection'
|
4
|
+
require 'zugzwang/connection/populate'
|
7
5
|
require 'thor'
|
8
6
|
|
9
7
|
module Zugzwang
|
10
8
|
class CLI < Thor
|
11
9
|
include Thor::Actions
|
12
10
|
|
13
|
-
method_option :
|
11
|
+
method_option :adapter, type: :string, required: true, default: 'sqlite', aliases: '--adapter'
|
14
12
|
desc 'create [DATABASE] *[ITEMS]', "Converts the metadata of PGN files from Lichess' game database to a database file."
|
15
13
|
def create(database = 'lichess', *items)
|
16
14
|
if items.empty?
|
@@ -18,57 +16,29 @@ module Zugzwang
|
|
18
16
|
return
|
19
17
|
end
|
20
18
|
|
21
|
-
|
22
|
-
|
23
|
-
override = false
|
24
|
-
|
25
|
-
loop do
|
26
|
-
if Zugzwang::EXTENSIONS.include?(extension) || override
|
27
|
-
|
28
|
-
begin
|
29
|
-
db = Sequel.sqlite("#{database}.#{extension}")
|
30
|
-
Zugzwang::Create[db, database, extension, items]
|
31
|
-
rescue Sequel::DatabaseConnectionError => e
|
32
|
-
puts "\n\e[1;91mERROR\e[0m: Directory \e[1m#{File.dirname(database)}\e[0m does not exist."
|
33
|
-
|
34
|
-
pass = false
|
35
|
-
until pass
|
36
|
-
response = ask("\e[1mPROMPT: \e[0mCreate directory \e[1m#{File.dirname(database)}\e[0m? [Y/n]")
|
37
|
-
if %w[Y y YES Yes yes].include? response
|
38
|
-
puts
|
39
|
-
empty_directory(File.dirname(database))
|
40
|
-
begin
|
41
|
-
db = Sequel.sqlite("#{database}.#{extension}")
|
42
|
-
Zugzwang::Create[db, database, extension, items]
|
43
|
-
rescue Sequel::DatabaseConnectionError
|
44
|
-
puts "\n\e[1;91mERROR\e[0m: Unable to create database with extension \e[1m#{extension}\e[0m"
|
45
|
-
end
|
46
|
-
pass = true
|
47
|
-
elsif %w[N n NO No no].include? response
|
48
|
-
pass = true
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
end
|
53
|
-
|
54
|
-
return
|
55
|
-
|
56
|
-
else
|
57
|
-
|
58
|
-
puts "\n\e[1;93mWARNING\e[0m: Extension argument should be one of [#{Zugzwang::EXTENSIONS*', '}]."
|
19
|
+
database_path = sanitize_path(database)
|
20
|
+
adapter = options[:adapter].to_sym
|
59
21
|
|
22
|
+
if Zugzwang::ADAPTERS.include?(adapter)
|
23
|
+
begin
|
24
|
+
Zugzwang::Connection.new(database_path, adapter).populate(items)
|
25
|
+
rescue Sequel::DatabaseConnectionError
|
26
|
+
puts "\n\e[1;91mERROR\e[0m: Directory \e[1m#{File.dirname(database)}\e[0m does not exist."
|
60
27
|
pass = false
|
61
28
|
until pass
|
62
|
-
response = ask("\e[1mPROMPT: \e[
|
29
|
+
response = ask("\e[1mPROMPT: \e[0mCreate directory \e[1m#{File.dirname(database_path)}\e[0m? [Y/n]")
|
63
30
|
if %w[Y y YES Yes yes].include? response
|
64
|
-
|
31
|
+
puts
|
32
|
+
empty_directory(File.dirname(database_path))
|
33
|
+
Zugzwang::Connection.new(database_path,adapter).populate(items)
|
65
34
|
pass = true
|
66
35
|
elsif %w[N n NO No no].include? response
|
67
|
-
|
36
|
+
pass = true
|
68
37
|
end
|
69
38
|
end
|
70
|
-
|
71
39
|
end
|
40
|
+
else
|
41
|
+
puts "\n\e[1;91mERROR\e[0m: Database adapter argument should be one of [#{Zugzwang::ADAPTERS*', '}]."
|
72
42
|
end
|
73
43
|
end
|
74
44
|
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
module Zugzwang
|
5
|
+
class Connection
|
6
|
+
def initialize(database_path, adapter)
|
7
|
+
@database_path = database_path
|
8
|
+
@adapter = adapter
|
9
|
+
case adapter
|
10
|
+
when :sqlite then @database = Sequel.sqlite("#{database_path}.sqlite3")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'ruby-progressbar'
|
2
|
+
require 'date'
|
3
|
+
require 'time'
|
4
|
+
require 'thor'
|
5
|
+
|
6
|
+
module Zugzwang
|
7
|
+
class Connection
|
8
|
+
def populate(items)
|
9
|
+
return unless create()
|
10
|
+
|
11
|
+
files = items.map{|i|File.file?(i) ? Dir[i] : Dir["#{i}/**/*.pgn"]}.flatten
|
12
|
+
puts if files.map{|f|File.file? f}.any?{|f|f==true}
|
13
|
+
|
14
|
+
files.each do |f|
|
15
|
+
record = {}
|
16
|
+
prev = ''
|
17
|
+
|
18
|
+
# Progress bar
|
19
|
+
lines = `sed -n '=' #{f} | wc -l`.to_i
|
20
|
+
puts "\e[90mProcessing file:\e[0m \e[1m#{f}\e[0m"
|
21
|
+
progress_bar = ProgressBar.create(
|
22
|
+
:title => 'Progress',
|
23
|
+
total: lines,
|
24
|
+
progress_mark: "\e[1;35m#{?#}\e[0m",
|
25
|
+
remainder_mark: ?.,
|
26
|
+
format: "%t: %p%% (Line: %c/%C) %B"
|
27
|
+
)
|
28
|
+
|
29
|
+
# Processing file
|
30
|
+
File.open(f,'r').each do |line|
|
31
|
+
if prev =~ /\A\[.*\Z/ && line =~ /\A[\n\r].*\Z/
|
32
|
+
@database[:games].insert(**record)
|
33
|
+
record = {}
|
34
|
+
prev = line
|
35
|
+
progress_bar.increment
|
36
|
+
next
|
37
|
+
end
|
38
|
+
if line =~ /\A([\n\r]|[0-9]).*\Z/
|
39
|
+
prev = line
|
40
|
+
progress_bar.increment
|
41
|
+
next
|
42
|
+
end
|
43
|
+
subbed = line.gsub(%r{[\r\n\[\]\"]},'')
|
44
|
+
field, value = subbed.split(' ', 2)
|
45
|
+
field = field.to_sym
|
46
|
+
if FIELDS[field]
|
47
|
+
record[field] = case FIELDS[field][:type]
|
48
|
+
when :string then value
|
49
|
+
when :integer then value.to_i
|
50
|
+
when :date then Date.parse(value)
|
51
|
+
when :time then value
|
52
|
+
end
|
53
|
+
end
|
54
|
+
prev = line
|
55
|
+
progress_bar.increment
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
case @adapter
|
60
|
+
when :sqlite
|
61
|
+
puts "\n\e[92mComplete\e[0m: Populated #{@adapter} database at \e[1m#{@database_path}.sqlite3\e[0m."
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def create
|
68
|
+
begin
|
69
|
+
@database.create_table :games do
|
70
|
+
primary_key :id
|
71
|
+
FIELDS.each do |name, options|
|
72
|
+
case options[:type]
|
73
|
+
when :string then String name, size: (options[:size]||255)
|
74
|
+
when :integer then Integer name
|
75
|
+
when :date then Date name
|
76
|
+
when :time then Time name, only_time: true
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
return true
|
81
|
+
rescue Sequel::DatabaseError
|
82
|
+
pass = false
|
83
|
+
until pass
|
84
|
+
puts "\n\e[1;91mERROR\e[0m: '\e[1mgames\e[0m' table already exists in the database."
|
85
|
+
response = Ask.new.msg("\e[1mPROMPT: \e[0mOverwrite table? (\e[1mEXISTING DATA WILL BE DELETED!\e[0m) [Y/n]")
|
86
|
+
if %w[Y y YES Yes yes].include? response
|
87
|
+
@database.create_table! :games do
|
88
|
+
primary_key :id
|
89
|
+
FIELDS.each do |name, options|
|
90
|
+
case options[:type]
|
91
|
+
when :string then String name, size: (options[:size]||255)
|
92
|
+
when :integer then Integer name
|
93
|
+
when :date then Date name
|
94
|
+
when :time then Time name, only_time: true
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
pass = true
|
99
|
+
return true
|
100
|
+
elsif %w[N n NO No no].include? response
|
101
|
+
pass = true
|
102
|
+
return false
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
class Ask < Thor
|
109
|
+
include Thor::Actions
|
110
|
+
def initialize() end
|
111
|
+
|
112
|
+
no_tasks do
|
113
|
+
def msg(message)
|
114
|
+
ask(message)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
data/lib/zugzwang/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zugzwang
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edwin Onuonga
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-07-
|
11
|
+
date: 2018-07-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -124,7 +124,8 @@ files:
|
|
124
124
|
- bin/zugzwang
|
125
125
|
- lib/zugzwang.rb
|
126
126
|
- lib/zugzwang/cli.rb
|
127
|
-
- lib/zugzwang/
|
127
|
+
- lib/zugzwang/connection/connection.rb
|
128
|
+
- lib/zugzwang/connection/populate.rb
|
128
129
|
- lib/zugzwang/helpers.rb
|
129
130
|
- lib/zugzwang/version.rb
|
130
131
|
- zugzwang.gemspec
|
data/lib/zugzwang/create.rb
DELETED
@@ -1,64 +0,0 @@
|
|
1
|
-
require 'ruby-progressbar'
|
2
|
-
require 'date'
|
3
|
-
require 'time'
|
4
|
-
|
5
|
-
module Zugzwang
|
6
|
-
module Create
|
7
|
-
def self.[](database_object, database, extension, items)
|
8
|
-
database_object.create_table :games do
|
9
|
-
primary_key :id
|
10
|
-
FIELDS.each do |name, options|
|
11
|
-
case options[:type]
|
12
|
-
when :string then String name, size: (options[:size]||255)
|
13
|
-
when :integer then Integer name
|
14
|
-
when :date then Date name
|
15
|
-
when :time then Time name, only_time: true
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
files = items.map{|item|
|
21
|
-
File.file?(item) ? Dir[item] : Dir["#{item}/**/*.pgn"]
|
22
|
-
}.flatten
|
23
|
-
|
24
|
-
puts if files.map{|f|File.file? f}.any?{|f|f==true}
|
25
|
-
|
26
|
-
files.each do |f|
|
27
|
-
record = {}
|
28
|
-
prev = ''
|
29
|
-
lines = `sed -n '=' #{f} | wc -l`.to_i
|
30
|
-
puts "\e[90mProcessing file:\e[0m \e[1m#{f}\e[0m"
|
31
|
-
progress_bar = ProgressBar.create(:title => 'Progress', total: lines, progress_mark: "\e[1;35m#{?#}\e[0m", remainder_mark: ?., format: "%t: %p%% (Line: %c/%C) %B")
|
32
|
-
File.open(f,'r').each do |line|
|
33
|
-
if prev =~ /\A\[.*\Z/ && line =~ /\A[\n\r].*\Z/
|
34
|
-
database_object[:games].insert(**record)
|
35
|
-
record = {}
|
36
|
-
prev = line
|
37
|
-
progress_bar.increment
|
38
|
-
next
|
39
|
-
end
|
40
|
-
if line =~ /\A([\n\r]|[0-9]).*\Z/
|
41
|
-
prev = line
|
42
|
-
progress_bar.increment
|
43
|
-
next
|
44
|
-
end
|
45
|
-
subbed = line.gsub(%r{[\r\n\[\]\"]},'')
|
46
|
-
field, value = subbed.split(' ', 2)
|
47
|
-
field = field.to_sym
|
48
|
-
if FIELDS[field]
|
49
|
-
record[field] = case FIELDS[field][:type]
|
50
|
-
when :string then value
|
51
|
-
when :integer then value.to_i
|
52
|
-
when :date then Date.parse(value)
|
53
|
-
when :time then value
|
54
|
-
end
|
55
|
-
end
|
56
|
-
prev = line
|
57
|
-
progress_bar.increment
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
puts "\n\e[92mComplete\e[0m: Saved #{extension} database at \e[1m#{database}.#{extension}\e[0m."
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|