anansi 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5485f4a8cdebb188f7216267be755f55ab898d5259b8e0cc0d14926779a9c4e0
+  data.tar.gz: 8c63b03367addfdef4d41bf62e835b358384612b7c63e2413fa03b4b943740f2
+SHA512:
+  metadata.gz: bfe9cb9fb95cbf7dc6a9edbfdf29542bcd2604c6788071f1c229435daf58620341cc1393017238c03280f8d17d92e0e64fe589624ebca7d50887d81796dad250
+  data.tar.gz: 70a253f8c9077f3603522b503ab3171132d8989a127b1641d629cf7003e8fa45131e86d2ff1d0520d6638e1bcc92fd277dc753b5134638ebf5a47ce1a95cdcce

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.0.7] - 2023-01-30
+
+- Initial release

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in anansi.gemspec
+gemspec
+
+gem 'sorbet', group: :development
+gem 'sorbet-runtime'
+gem 'sqlite3'
+gem 'tapioca', require: false, group: :development

data/Gemfile.lock

@@ -0,0 +1,63 @@
+PATH
+  remote: .
+  specs:
+    anansi (0.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    diff-lcs (1.5.0)
+    netrc (0.11.0)
+    parallel (1.22.1)
+    parser (3.1.3.0)
+      ast (~> 2.4.1)
+    rbi (0.0.16)
+      ast
+      parser (>= 2.6.4.0)
+      sorbet-runtime (>= 0.5.9204)
+      unparser
+    sorbet (0.5.10568)
+      sorbet-static (= 0.5.10568)
+    sorbet-runtime (0.5.10568)
+    sorbet-static (0.5.10568-universal-darwin-21)
+    sorbet-static-and-runtime (0.5.10568)
+      sorbet (= 0.5.10568)
+      sorbet-runtime (= 0.5.10568)
+    spoom (1.1.13)
+      sorbet (>= 0.5.9204)
+      sorbet-runtime (>= 0.5.9204)
+      thor (>= 0.19.2)
+    sqlite3 (1.5.3-arm64-darwin)
+    tapioca (0.10.3)
+      bundler (>= 1.17.3)
+      netrc (>= 0.11.0)
+      parallel (>= 1.21.0)
+      rbi (~> 0.0.0, >= 0.0.16)
+      sorbet-static-and-runtime (>= 0.5.9892)
+      spoom (~> 1.1.0, >= 1.1.11)
+      thor (>= 1.2.0)
+      yard-sorbet
+    thor (1.2.1)
+    unparser (0.6.5)
+      diff-lcs (~> 1.3)
+      parser (>= 3.1.0)
+    webrick (1.7.0)
+    yard (0.9.28)
+      webrick (~> 1.7.0)
+    yard-sorbet (0.7.0)
+      sorbet-runtime (>= 0.5)
+      yard (>= 0.9)
+
+PLATFORMS
+  arm64-darwin-21
+
+DEPENDENCIES
+  anansi!
+  sorbet
+  sorbet-runtime
+  sqlite3
+  tapioca
+
+BUNDLED WITH
+   2.3.22

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Sutro Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# Anansi - A hybrid Ruby Set using memory and Disk (using sqlite3) for large sized tasks
+
+> He lifted the pot over his head and threw it on the ground. The pot crashed on the ground and the wisdom blew far and wide all over the earth. And this is how wisdom came to the world (or your disk).
+
+**- Kiren Babal** ([Anansi and the Wisdom Pot](https://www.differenttruths.com/literature/short-story/anansi-and-the-wisdom-pot/))
+
+## Why?
+
+A Ruby Set in memory isn't great for huge tasks. But a disk based Set is too slow for everything. We need the best of both worlds.
+
+## What?
+
+Data structures that use constant memory by spilling to disk after crossing a size threshold.
+
+Currently the only supported data structure is `AppendSet`.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add anansi
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install anansi
+
+## Usage
+Add items to an `AppendSet`:
+```ruby
+append_set = Anansi::AppendSet.new
+append_set.add(['foo', 'bar', 'buzz'])
+```
+
+Check if an item exists in an `AppendSet`:
+```ruby
+append_set.include? 'foo'
+```
+
+Get the size of an `AppendSet`:
+```ruby
+append_set.size
+```
+
+(If you need other data structures, stay tuned and [watch our org](https://github.com/sutrolabs).)
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+Feedback
+--------
+[Source code available on Github](https://github.com/sutrolabs/anansi). Feedback and pull requests are greatly appreciated. Let us know if we can improve this.
+
+From
+-----------
+:wave: The folks at [Census](http://getcensus.com) originally put this together. Have data? We'll sync your data warehouse with your CRM and the customer success apps critical to your team.

data/lib/anansi/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Anansi
+  VERSION = "0.0.0"
+end

data/lib/anansi.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+require "sqlite3"
+require "sorbet-runtime"
+
+module Anansi
+  # An append-only implementation of parts of Set from the stdlib that uses
+  # constant memory regardless of the set size by spilling data to disk (using
+  # SQLite) once the set size crosses some threshold.
+  class AppendSet
+    # Interface to abstract over Set / SQLite
+    module Store
+      extend T::Sig
+      extend T::Helpers
+      extend T::Generic
+
+      interface!
+
+      Item = type_member { { upper: Object } }
+
+      T::Sig::WithoutRuntime.sig { abstract.returns(Integer) }
+      def size; end
+
+      T::Sig::WithoutRuntime.sig { abstract.params(item: Item).returns(T::Boolean) }
+      def include?(item); end
+
+      T::Sig::WithoutRuntime.sig { abstract.params(items: T::Enumerable[Item]).void }
+      def add(items); end
+    end
+
+    # Implementation for Set
+    class SetStore
+      extend T::Sig
+      extend T::Generic
+
+      Item = type_member { { upper: Object } }
+
+      include Store
+
+      T::Sig::WithoutRuntime.sig { void }
+      def initialize
+        @set = T.let(Set.new, T::Set[Item])
+      end
+
+      T::Sig::WithoutRuntime.sig { returns(T::Enumerable[Item]) }
+      def all
+        @set
+      end
+
+      T::Sig::WithoutRuntime.sig { override.returns(Integer) }
+      def size
+        @set.size
+      end
+
+      T::Sig::WithoutRuntime.sig { override.params(item: Item).returns(T::Boolean) }
+      def include?(item)
+        @set.include?(item)
+      end
+
+      T::Sig::WithoutRuntime.sig { override.params(items: T::Enumerable[Item]).void }
+      def add(items)
+        items.each { |item| @set.add(item) }
+      end
+    end
+
+    # Implementation for SQLite
+    class SQLiteStore
+      extend T::Sig
+      extend T::Generic
+
+      Item = type_member { { upper: Object } }
+
+      include Store
+
+      T::Sig::WithoutRuntime.sig { void }
+      def initialize
+        # Both +@tempfile+ and +@sqlite+ get automatically cleaned up (closed,
+        # deleted from disk) when this object goes out of scope. Worst-case, files
+        # on disk should get reaped at process exit or dyno reboot.
+        @tempfile = T.let(Tempfile.new(%w[AppendSetSQLiteStore .sqlite3]), Tempfile)
+
+        # We immediately unlink the file so that it is guaranteed to be removed
+        # from disk on process exit (by the OS). In testing, I've found that Ruby
+        # guarantee of unlinking the file in the finalizer doesn't always seem to
+        # succeed; not sure why. +unlink+ is safe here because the @sqlite
+        # instance variable holds an open file handle - as long as a file has an
+        # open handle on a unix-like OS, it won't actually be deleted until that
+        # file handle is closed (though it does become invisible to commands like
+        # `ls`).
+        @sqlite = T.let(SQLite3::Database.new(@tempfile), SQLite3::Database)
+        @tempfile.unlink
+
+        # This is a non-persistent db so make it go fast
+        # Partially based on https://github.com/avinassh/fast-sqlite3-inserts
+        @sqlite.execute "pragma synchronous=off"
+        @sqlite.execute "pragma journal_mode=off"
+        @sqlite.execute "pragma locking_mode=exclusive"
+        @sqlite.execute "pragma temp_store=memory"
+
+        # The single table that will store data
+        @sqlite.execute "create table items (item text primary key)"
+
+        # Prepare our three statements
+        @count_statement = T.let(@sqlite.prepare("select count(item) from items"), SQLite3::Statement)
+        @add_statement =
+          T.let(@sqlite.prepare("insert into items values (?) on conflict (item) do nothing"), SQLite3::Statement)
+        @include_statement =
+          T.let(@sqlite.prepare("select exists(select 1 from items where item = ?)"), SQLite3::Statement)
+      end
+
+      T::Sig::WithoutRuntime.sig { override.returns(Integer) }
+      def size
+        result = @count_statement.execute
+        result.first.first
+      end
+
+      T::Sig::WithoutRuntime.sig { override.params(item: Item).returns(T::Boolean) }
+      def include?(item)
+        result = @include_statement.execute(marshal(item))
+        result.first.first == 1
+      end
+
+      T::Sig::WithoutRuntime.sig { override.params(items: T::Enumerable[Item]).void }
+      def add(items)
+        # transaction should improve bulk insert performance
+        @sqlite.execute "begin"
+        items.each { |item| @add_statement.execute(marshal(item)) }
+        @sqlite.execute "commit"
+      end
+
+      private
+
+      T::Sig::WithoutRuntime.sig { params(item: Item).returns(String) }
+      def marshal(item)
+        Base64.encode64(Marshal.dump(item))
+      end
+    end
+
+    # Facade that implements Store by proxying to SetStore or SQLiteStore. The
+    # spillover logic is also included here, as well as an optimization for +any?+
+    extend T::Sig
+    extend T::Generic
+    include Store
+    Item = type_member { { upper: Object } }
+
+    # This is pretty arbitrary, but it seems to correlate well with efficient
+    # sqlite transaction performance and it gives us enough granularity in
+    # checkpointing to see if we should spill
+    ADD_BATCH_SIZE = 50_000
+
+    # This is pretty arbitrary, but assuming that each key is ~200 bytes in memory
+    # (a conservative estimate), it means that we'll never burn more than 100MB of
+    # RAM on this set before we go to disk. Ideally we could express this as a byte
+    # threshold, not an item threshold - I couldn't find an easy / cheap way to do
+    # that in Ruby.
+    SPILL_THRESHOLD = 500_000
+
+    T::Sig::WithoutRuntime.sig { void }
+    def initialize
+      @store = T.let(SetStore[Item].new, Store[Item])
+      @spilled = T.let(false, T::Boolean)
+    end
+
+    T::Sig::WithoutRuntime.sig { returns(T::Boolean) }
+    def any?
+      if @spilled
+        # We must be non-empty if we've spilled to SQLite, so avoid an expensive
+        # call to the DB to check
+        true
+      else
+        set_store = T.cast(@store, SetStore[Item])
+        set_store.all.any?
+      end
+    end
+
+    T::Sig::WithoutRuntime.sig { override.returns(Integer) }
+    def size
+      @store.size
+    end
+
+    T::Sig::WithoutRuntime.sig { override.params(item: Item).returns(T::Boolean) }
+    def include?(item)
+      @store.include?(item)
+    end
+
+    T::Sig::WithoutRuntime.sig { override.params(items: T::Enumerable[Item]).void }
+    def add(items)
+      items.each_slice(ADD_BATCH_SIZE) do |slice|
+        @store.add(slice)
+        spill_if_needed!
+      end
+    end
+
+    private
+
+    T::Sig::WithoutRuntime.sig { void }
+    def spill_if_needed!
+      return if @spilled
+      return if @store.size < SPILL_THRESHOLD
+
+      set_store = T.cast(@store, SetStore[Item])
+      sqlite_store = SQLiteStore.new
+      sqlite_store.add(set_store.all)
+      @store = sqlite_store
+      @spilled = true
+    end
+  end
+end

data/sorbet/config

@@ -0,0 +1,3 @@
+--dir
+.
+--ignore=vendor/