airspace 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +8 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +11 -0
- data/.ruby-version +1 -0
- data/.travis.yml +22 -0
- data/CHANGELOG.md +3 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +104 -0
- data/Guardfile +16 -0
- data/LICENSE +7 -0
- data/README.md +220 -0
- data/airspace.gemspec +33 -0
- data/bin/console +15 -0
- data/lib/airspace.rb +10 -0
- data/lib/airspace/airspace.rb +50 -0
- data/lib/airspace/chunker.rb +47 -0
- data/lib/airspace/dataset.rb +79 -0
- data/lib/airspace/has_metadata.rb +24 -0
- data/lib/airspace/info_keys.rb +17 -0
- data/lib/airspace/key.rb +35 -0
- data/lib/airspace/metadata.rb +44 -0
- data/lib/airspace/reader.rb +92 -0
- data/lib/airspace/serializer.rb +42 -0
- data/lib/airspace/store.rb +115 -0
- data/lib/airspace/version.rb +12 -0
- data/spec/airspace/airspace_spec.rb +177 -0
- data/spec/airspace/chunker_spec.rb +111 -0
- data/spec/airspace/store_spec.rb +21 -0
- data/spec/spec_helper.rb +33 -0
- metadata +190 -0
data/bin/console
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'bundler/setup'
|
5
|
+
require 'airspace'
|
6
|
+
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
9
|
+
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
11
|
+
# require "pry"
|
12
|
+
# Pry.start
|
13
|
+
|
14
|
+
require 'irb'
|
15
|
+
IRB.start
|
data/lib/airspace.rb
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'airspace/airspace'
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'forwardable'
|
11
|
+
require 'json'
|
12
|
+
require 'securerandom'
|
13
|
+
require 'time'
|
14
|
+
|
15
|
+
require_relative 'key'
|
16
|
+
require_relative 'info_keys'
|
17
|
+
require_relative 'chunker'
|
18
|
+
require_relative 'metadata'
|
19
|
+
require_relative 'has_metadata'
|
20
|
+
require_relative 'dataset'
|
21
|
+
require_relative 'serializer'
|
22
|
+
require_relative 'reader'
|
23
|
+
require_relative 'store'
|
24
|
+
|
25
|
+
# Top-level namespace for primary public API.
|
26
|
+
module Airspace
|
27
|
+
class << self
|
28
|
+
def set(client, id: nil, data: {}, pages: [], options: {})
|
29
|
+
::Airspace::Dataset.new(
|
30
|
+
client,
|
31
|
+
id: id,
|
32
|
+
data: data,
|
33
|
+
pages: pages,
|
34
|
+
options: options
|
35
|
+
).save.id
|
36
|
+
end
|
37
|
+
|
38
|
+
def get(client, id, options: {})
|
39
|
+
::Airspace::Reader.find_by_id(client, id, options: options)
|
40
|
+
end
|
41
|
+
|
42
|
+
def del(client, id, options: {})
|
43
|
+
reader = get(client, id, options: options)
|
44
|
+
|
45
|
+
return false unless reader
|
46
|
+
|
47
|
+
reader.delete
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# Chunking here is defined as: taking an array of pages and grouping them into groups of pages
|
12
|
+
# (chunks) in order to find a middle-ground of server-side page and entire dataset fetches.
|
13
|
+
class Chunker
|
14
|
+
Location = Struct.new(:chunk_index, :page_index)
|
15
|
+
Chunk = Struct.new(:chunk_index, :page_index_start, :page_index_end)
|
16
|
+
|
17
|
+
attr_reader :pages_per_chunk
|
18
|
+
|
19
|
+
def initialize(pages_per_chunk)
|
20
|
+
raise ArgumentError unless pages_per_chunk.positive?
|
21
|
+
|
22
|
+
@pages_per_chunk = pages_per_chunk
|
23
|
+
end
|
24
|
+
|
25
|
+
def count(page_total)
|
26
|
+
(page_total / pages_per_chunk.to_f).ceil
|
27
|
+
end
|
28
|
+
|
29
|
+
def each(page_total)
|
30
|
+
return enum_for(:each, page_total) unless block_given?
|
31
|
+
|
32
|
+
(0...count(page_total)).each do |chunk_index|
|
33
|
+
page_index_start = chunk_index * pages_per_chunk
|
34
|
+
page_index_end = page_index_start + pages_per_chunk - 1
|
35
|
+
|
36
|
+
yield Chunk.new(chunk_index, page_index_start, page_index_end)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def locate(index)
|
41
|
+
chunk_index = (index / pages_per_chunk.to_f).floor
|
42
|
+
page_index = index % pages_per_chunk
|
43
|
+
|
44
|
+
Location.new(chunk_index, page_index)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# This is the main input class that can persist data.
|
12
|
+
class Dataset
|
13
|
+
include ::Airspace::InfoKeys
|
14
|
+
include ::Airspace::HasMetadata
|
15
|
+
|
16
|
+
attr_reader :client,
|
17
|
+
:data,
|
18
|
+
:id,
|
19
|
+
:pages,
|
20
|
+
:prefix,
|
21
|
+
:serializer
|
22
|
+
|
23
|
+
def initialize(client, id: nil, data: {}, pages: [], options: {})
|
24
|
+
raise ArgumentError, 'client is required' unless client
|
25
|
+
|
26
|
+
@client = client
|
27
|
+
@data = data || {}
|
28
|
+
@id = id || SecureRandom.uuid
|
29
|
+
@pages = pages || []
|
30
|
+
@prefix = options[:prefix].to_s
|
31
|
+
@serializer = options[:serializer] || ::Airspace::Serializer.new
|
32
|
+
|
33
|
+
@metadata = ::Airspace::Metadata.new(
|
34
|
+
expires_in_seconds: options[:expires_in_seconds],
|
35
|
+
page_count: pages.length,
|
36
|
+
pages_per_chunk: options[:pages_per_chunk]
|
37
|
+
)
|
38
|
+
|
39
|
+
freeze
|
40
|
+
end
|
41
|
+
|
42
|
+
def save
|
43
|
+
store.persist(key, info_hash, chunks, expires_in_seconds)
|
44
|
+
|
45
|
+
self
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def info_hash
|
51
|
+
{}.tap do |hash|
|
52
|
+
hash[DATA_KEY] = serializer.serialize_data(data)
|
53
|
+
hash[METADATA_KEY] = metadata.to_json
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def chunks
|
58
|
+
chunks = []
|
59
|
+
|
60
|
+
chunker.each(page_count) do |chunk|
|
61
|
+
chunk_data = pages[chunk.page_index_start..chunk.page_index_end]
|
62
|
+
|
63
|
+
chunks << chunk_data.map do |page|
|
64
|
+
page.map { |row| serializer.serialize_row(row) }
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
chunks
|
69
|
+
end
|
70
|
+
|
71
|
+
def store
|
72
|
+
::Airspace::Store.new(client)
|
73
|
+
end
|
74
|
+
|
75
|
+
def key
|
76
|
+
::Airspace::Key.new(id, prefix: prefix)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# This mix-in allows for classes to be composed of a Metadata instance
|
12
|
+
module HasMetadata
|
13
|
+
extend Forwardable
|
14
|
+
|
15
|
+
attr_reader :metadata
|
16
|
+
|
17
|
+
def_delegators :metadata,
|
18
|
+
:chunk_count,
|
19
|
+
:chunker,
|
20
|
+
:expires_in_seconds,
|
21
|
+
:page_count,
|
22
|
+
:pages_per_chunk
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# Holds shared data among how to internally access data.
|
12
|
+
module InfoKeys
|
13
|
+
DATA_KEY = 'd'
|
14
|
+
METADATA_KEY = 'm'
|
15
|
+
SEPARATOR_CHAR = ':'
|
16
|
+
end
|
17
|
+
end
|
data/lib/airspace/key.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# This class understands how to build keys and subkeys for storing data inside Redis.
|
12
|
+
class Key
|
13
|
+
SEPARATOR_CHAR = ':'
|
14
|
+
|
15
|
+
private_constant :SEPARATOR_CHAR
|
16
|
+
|
17
|
+
attr_reader :id, :prefix
|
18
|
+
|
19
|
+
def initialize(id, prefix: '')
|
20
|
+
@id = id.to_s
|
21
|
+
@prefix = prefix.to_s
|
22
|
+
end
|
23
|
+
|
24
|
+
def root
|
25
|
+
return id if prefix.empty?
|
26
|
+
|
27
|
+
[prefix, id].join(SEPARATOR_CHAR)
|
28
|
+
end
|
29
|
+
alias to_s root
|
30
|
+
|
31
|
+
def chunk(index)
|
32
|
+
[root, index].join(SEPARATOR_CHAR)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# Metdata is 'data about a dataset.' These are key pieces of information we need to store with
|
12
|
+
# the data then later retrieve with the data.
|
13
|
+
class Metadata
|
14
|
+
DEFAULT_PAGES_PER_CHUNK = 5
|
15
|
+
|
16
|
+
attr_reader :expires_in_seconds,
|
17
|
+
:page_count,
|
18
|
+
:pages_per_chunk
|
19
|
+
|
20
|
+
def initialize(expires_in_seconds: nil, page_count: 0, pages_per_chunk: DEFAULT_PAGES_PER_CHUNK)
|
21
|
+
@expires_in_seconds = expires_in_seconds ? expires_in_seconds.to_i : nil
|
22
|
+
@page_count = page_count.to_i
|
23
|
+
@pages_per_chunk = pages_per_chunk ? pages_per_chunk.to_i : DEFAULT_PAGES_PER_CHUNK
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
|
28
|
+
def chunker
|
29
|
+
::Airspace::Chunker.new(pages_per_chunk)
|
30
|
+
end
|
31
|
+
|
32
|
+
def chunk_count
|
33
|
+
chunker.count(page_count)
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_json
|
37
|
+
{
|
38
|
+
expires_in_seconds: expires_in_seconds,
|
39
|
+
page_count: page_count,
|
40
|
+
pages_per_chunk: pages_per_chunk
|
41
|
+
}.to_json
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# This is the main class that knows how to fetch and interpret the dataset.
|
12
|
+
# It is optimized for chunking/paging and allows you to only pull back specific
|
13
|
+
# pages (if desired.)
|
14
|
+
class Reader
|
15
|
+
extend Forwardable
|
16
|
+
extend ::Airspace::InfoKeys
|
17
|
+
include ::Airspace::HasMetadata
|
18
|
+
|
19
|
+
class << self
|
20
|
+
def find_by_id(client, id, options: {})
|
21
|
+
key = ::Airspace::Key.new(id, prefix: options[:prefix])
|
22
|
+
serializer = options[:serializer] || ::Airspace::Serializer.new
|
23
|
+
hash = fetch_and_transform(client, key, serializer)
|
24
|
+
return nil unless hash
|
25
|
+
|
26
|
+
metadata_args = hash[METADATA_KEY].map { |k, v| [k.to_sym, v] }.to_h
|
27
|
+
|
28
|
+
new(
|
29
|
+
client,
|
30
|
+
data: hash[DATA_KEY],
|
31
|
+
key: key,
|
32
|
+
metadata: ::Airspace::Metadata.new(metadata_args),
|
33
|
+
serializer: serializer
|
34
|
+
)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def fetch_and_transform(client, key, serializer)
|
40
|
+
hash = ::Airspace::Store.new(client).retrieve(key)
|
41
|
+
return nil unless hash
|
42
|
+
|
43
|
+
{}.tap do |h|
|
44
|
+
h[DATA_KEY] = serializer.deserialize_data(hash[DATA_KEY])
|
45
|
+
h[METADATA_KEY] = JSON.parse(hash[METADATA_KEY])
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
attr_reader :client,
|
51
|
+
:data,
|
52
|
+
:key,
|
53
|
+
:metadata,
|
54
|
+
:serializer
|
55
|
+
|
56
|
+
def_delegators :key, :id
|
57
|
+
|
58
|
+
def initialize(client, data:, key:, metadata:, serializer:)
|
59
|
+
@client = client
|
60
|
+
@key = key
|
61
|
+
@data = data
|
62
|
+
@metadata = metadata
|
63
|
+
@serializer = serializer
|
64
|
+
|
65
|
+
freeze
|
66
|
+
end
|
67
|
+
|
68
|
+
def pages
|
69
|
+
store.chunks(key, chunk_count).map do |chunk|
|
70
|
+
chunk.map { |r| serializer.deserialize_row(r) }
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def page(number)
|
75
|
+
page_index = number - 1
|
76
|
+
location = chunker.locate(page_index)
|
77
|
+
chunk = store.chunk(key, location.chunk_index)
|
78
|
+
|
79
|
+
chunk[location.page_index].map { |r| serializer.deserialize_row(r) }
|
80
|
+
end
|
81
|
+
|
82
|
+
def delete
|
83
|
+
store.delete(key, page_count)
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
def store
|
89
|
+
::Airspace::Store.new(client)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2019-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Airspace
|
11
|
+
# This class dictates how data is stored and retrieved. You can subclass this and
|
12
|
+
# change its implementation that suits your overall data/space requirements.
|
13
|
+
class Serializer
|
14
|
+
def serialize_data(obj)
|
15
|
+
json_serialize(obj)
|
16
|
+
end
|
17
|
+
|
18
|
+
def deserialize_data(json)
|
19
|
+
json_deserialize(json)
|
20
|
+
end
|
21
|
+
|
22
|
+
def serialize_row(obj)
|
23
|
+
json_serialize(obj)
|
24
|
+
end
|
25
|
+
|
26
|
+
def deserialize_row(json)
|
27
|
+
json_deserialize(json)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def json_deserialize(json)
|
33
|
+
return nil unless json
|
34
|
+
|
35
|
+
JSON.parse(json)
|
36
|
+
end
|
37
|
+
|
38
|
+
def json_serialize(obj)
|
39
|
+
obj.to_json
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|