jekyll-sqlite 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f14ced1e5bddd23e8aca1460f7acc53449b2d19fbf3bb67eb97f55f194facc95
-  data.tar.gz: f2a21e3b7d89d2d8d476c71ee2876025037009eb311df7d64fb1b9e2dca5cac3
+  metadata.gz: fbf075a895e90cae8f3b5d69299089ad2f2381b9ba255348b7f517c6ef03f8a0
+  data.tar.gz: 920c6c9292ead41d747e88a8cb6ab5e5a1a85d54d19781d3b675a316a90c3e98
 SHA512:
-  metadata.gz: 50c29d5008eb35fd2ed1fd6d9b6acf7cca7c1fcc0b5e3b4933506f0a29875118c02934ccbd2cec94f3676d7276f71527803bb4324467b36a29dcddfe8b8d2064
-  data.tar.gz: '09b4bd31b5fe04534892df1e5a229e3aca26cc447a147f2cf43d5e306a4c14c30156505b08481c78b89c1216b7e0aec6ba54ee3979ab002363d46dec3fc8f11a'
+  metadata.gz: 769762e5786644f055002eae18dd443e12989242f24f5ce79cbd03dcea235bf73cc7e6b4c6a70b30dac024d03ca03e84fbe5f16ef5cdf9289cd930b8ccbeedf4
+  data.tar.gz: 0c950eca035285304de7900e780b41e769340083b9966c2a01df02a4b10330bc8917cbd63be53f84852ceb1e8dfb7f7d4845c73c89afa0fa8f4719c8dfad82fb

data/.rubocop.yml

@@ -2,7 +2,12 @@ require: rubocop-rake
 AllCops:
   TargetRubyVersion: 3.0
   NewCops: enable
-
+  Exclude:
+    - "node_modules/**/*"
+    - "tmp/**/*"
+    - "vendor/**/*"
+    - ".git/**/*"
+    - "test/_plugins/jekyll_sqlite_generator.rb"
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes
@@ -15,4 +20,4 @@ Layout/LineLength:
   Max: 120
 
 Metrics/AbcSize:
-  Max: 20
+  Max: 15

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.1.4] - 2024-07-17
+- Per-page queries are now supported via a `sqlite` config block in the front matter.
+- Documents support for existing site data being used within queries.
+- Code cleanup and refactoring.
+- Don't disable journaling on the database.
+
 ## [0.1.3] - 2024-07-02
 - First functional version
 - Adds tests

data/Gemfile

@@ -6,7 +6,12 @@ source "https://rubygems.org"
 gemspec
 
 # These are development dependencies
-gem "rake", "~> 13.0"
 gem "jekyll", "~> 4.0"
+gem "rake", "~> 13.0"
 gem "rubocop", "~> 1.21"
 gem "rubocop-rake", "~> 0.6.0"
+
+# Ruby 3.4 preparedness
+gem "base64", "~> 0.2.0"
+gem "bigdecimal", "~> 3.1"
+gem "csv", "~> 3.3"

data/README.md

@@ -2,7 +2,8 @@
 
 A Jekyll generator plugin to lets you use SQLite database instead of data files as a data source. It lets you easily create APIs and websites from a SQLite database, by linking together a database file, your template, and the relevant queries.
 
-It additionally supports nested queries, so that you can use the rows of `site.data.items` as bind_params for your nested query.
+It supports site-level queries, per-page queries, and prepared queries that can
+use existing data (possibly generated via more queries) as parameters.
 
 [![Continuous Integration](https://github.com/captn3m0/jekyll-sqlite/actions/workflows/main.yml/badge.svg)](https://github.com/captn3m0/jekyll-sqlite/actions/workflows/main.yml) [![Gem Version](https://badge.fury.io/rb/jekyll-sqlite.svg)](https://badge.fury.io/rb/jekyll-sqlite)
 
@@ -31,73 +32,38 @@ the `test` directory for a functional example with the [Northwind database](http
 ```yml
 ...
 sqlite:
-  - data: orders
-    file: &db "_db/northwind.db"
-    query: SELECT * from Orders
   - data: customers
     file: *db
     query: SELECT * from Customers
-  - data: categories
-    file: *db
-    query: SELECT CategoryID, CategoryName, Description FROM Categories
-  # Note that the CategoryID parameter in the query is coming from site.data.categories[].CategoryID
-  # which was picked up in the previous query
-  - data: categories.products
-    file: *db
-    query: SELECT ProductID, ProductName FROM Products WHERE Products.CategoryID=:CategoryID
 ```
 
 Then, you can use the `site.data` attributes accordingly:
 
 ```liquid
-{{ site.data.categories | jsonify }}
-```
-
-## Generating Pages
-
-It works well with the `datapage_gen` plugin:
-
-See the [datapage_gen](https://github.com/avillafiorita/jekyll-datapage_gen) docs for more details.
-
-Here's a sample configuration:
-
-```yaml
-# This will automatically generate a file for each restaurant
-# restaurants/#{id}.html file
-# with the layout `_layouts/restaurant.html`
-# and page.id, page.name, page.active set
-# and page.title set to restaurant name
-sqlite:
-  restaurants:
-    file: _db/reviews.db
-    sql: SELECT id, name, last_review_date > 1672531200 as active, address FROM restaurants;
-page_gen:
-  - data: restaurants
-    template: restaurant
-    name: id
-    title: name
-    filter: active
+{{ site.data.customers | jsonify }}
 ```
 
-## Nested Queries
+## Prepared Queries
 
-You can use the rows of `site.data.items` as bind_params for your nested query. For this to work against
-data generated by the plugin, the configuration order must be correct, so you need `site.data.items` above `site.data.items.nested` in your configuration.
+This plugin supports prepared queries with parameter binding. This lets you
+use existing data from a previous query, or some other source (such as
+`site.data.*` or `page.*`) as a parameter in your query.
 
-Say you have a YAML file defining your items (`data/items.yaml`):
+Say you have a YAML file defining your items (`data/books.yaml`):
 
 ```yaml
 - id: 31323952-2708-42dc-a995-6006a23cbf00
-  name: Item 1
+  name: Time Travel with a Rubber Band
 - id: 5c8e67a0-d490-4743-b5b8-8e67bd1f95a2
-  name: Item 2
+  name: The Art of Cache Invalidation
 ```
 and the prices for the items in your SQLite database, the following configuration will enrich the `items` array with the price:
 
 ```yaml
 sql:
-- data: site.data.items.meta
-  query: SELECT price,author FROM pricing WHERE id =:id
+- data: items.books
+  query: SELECT price, author FROM pricing WHERE id =:id
+  db: books.db
 ```
 This would allow the following Liquid loop to be written:
 
@@ -107,23 +73,62 @@ This would allow the following Liquid loop to be written:
 {% endfor %}
 ```
 
-This works well with `results_as_configuration` as well.
+## Per Page Queries
+
+The exact same syntax can be used on a per-page basis to generate data within
+each page. This is helpful for keeping page-specific queries within the page
+itself. Here's an example:
 
 ```yaml
-sql:
-- data: site.data.items.meta
-  query: SELECT price,author FROM pricing WHERE id =:id
-  results_as_hash: false
+---
+FeaturedSupplierID: 2
+sqlite:
+  - data: suppliers
+    file: "_db/northwind.db"
+    query: "SELECT CompanyName, SupplierID FROM suppliers ORDER BY SupplierID"
+  - data: suppliers.products
+    # This is a prepared query, where SupplierID is coming from the previous query.
+    file: "_db/northwind.db"
+    query: "SELECT ProductName, CategoryID,UnitPrice FROM products WHERE SupplierID = :SupplierID"
+  # :FeaturedSupplierID is picked up automatically from the page frontmatter.
+  - data: FeaturedSupplier
+    file: "_db/northwind.db"
+    query: "SELECT * SupplierID = :FeaturedSupplierID"
+---
+{{page.suppliers|jsonify}}
 ```
 
-The following also renders the price and author:
+This will generate a `page.suppliers` array with all the suppliers, and a `page.FeaturedSupplier` object with the details of the featured supplier.
 
-```liquid
-{% for item in site.data.items %}
-  {{item.meta[0]}}, {{item.meta[1]}}
-{% endfor %}
+Each supplier will have a `products` array with all the products for that supplier.
+
+## Generating Pages
+
+It works well with the `datapage_gen` plugin:
+
+See the [datapage_gen](https://github.com/avillafiorita/jekyll-datapage_gen) docs for more details.
+
+Here's a sample configuration:
+
+```yaml
+sqlite:
+  restaurants:
+    file: _db/reviews.db
+    sql: SELECT id, name, last_review_date > 1672531200 as active, address FROM restaurants;
+page_gen:
+  - data: restaurants
+    template: restaurant
+    name: id
+    title: name
+    filter: active
 ```
 
+This will automatically generate a file for each restaurant
+restaurants/#{id}.html file with the layout `_layouts/restaurant.html` and page.id, page.name, page.active set and page.title set to restaurant name
+
+Note that the `datapage_gen` plugin will run _after_ the `jekyll-sqlite` plugin,
+if you generate any pages with per-page queries, these queries will not execute.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -2,7 +2,67 @@
 
 require "bundler/gem_tasks"
 require "rubocop/rake_task"
+require "jekyll"
+require "sqlite3"
 
 RuboCop::RakeTask.new
 
+def assert(cond, msg = "Assertion Failed")
+  raise msg unless cond
+end
+
+def query_db(query)
+  db = SQLite3::Database.new "_db/northwind.db"
+  db.results_as_hash = true
+  results = db.execute query
+  results[0]
+end
+
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Metrics/MethodLength
+def validate_json
+  file = "_site/data.json"
+  data = JSON.parse(File.read(file))
+  assert data["orders"].size == 53, "Expected 53 orders, got #{data["orders"].size}"
+  assert data["customers"].size == 93, "Expected 93 customers, got #{data["customers"].size}"
+  assert data["categories"].size == 8, "Expected 93 categories, got #{data["categories"].size}"
+  assert data["orders"][0] == query_db("SELECT * FROM Orders LIMIT 1"), "Order Fetch Failed"
+  assert data["customers"][0] == query_db("SELECT * FROM Customers LIMIT 1"), "Customer Fetch Failed"
+  assert data["customers"][0] == query_db("SELECT * FROM Customers LIMIT 1"), "Customer Fetch Failed"
+  assert data["categories"][0]["products"].size == 12, "Products don't match"
+  data["categories"][0]["products"].each do |p|
+    assert p["CategoryID"] == 1, "CategoryID doesn't match"
+  end
+  assert data["delayedOrders"].size == 17
+  assert data["delayedOrders"][0]["OrderID"] == 10_249
+end
+
+def validate_page_json
+  file = "_site/suppliers.json"
+  read_data = JSON.parse(File.read(file))
+  data = read_data["allSuppliers"]
+  assert data.size == 29, "Expected 29 suppliers, got #{data.size}"
+  r = query_db("SELECT CompanyName, SupplierID FROM Suppliers ORDER BY SupplierID LIMIT 1")
+  assert r["CompanyName"] == data[0]["CompanyName"], "Company Name doesn't match"
+  assert r["SupplierID"] == data[0]["SupplierID"], "Supplier ID doesn't match"
+  assert data[0]["products"].size == 3, "Products don't match"
+  assert data[0]["products"][0]["ProductName"] == "Chai"
+  assert data[0]["products"][1]["ProductName"] == "Chang"
+  assert data[0]["products"][2]["ProductName"] == "Aniseed Syrup"
+
+  # Focus Supplier - this uses the data from the page front-matter to prepare a query
+  fs = query_db("SELECT * FROM Suppliers WHERE SupplierID = 6")
+  assert read_data["focusSupplier"][0] == fs, "Focus Supplier doesn't match"
+end
+# rubocop:enable Metrics/AbcSize
+# rubocop:enable Metrics/MethodLength
+
 task default: :rubocop
+
+desc "Build Test Site"
+task :test do
+  Dir.chdir("test")
+  Jekyll::Site.new(Jekyll.configuration).process
+  validate_json
+  validate_page_json
+end

data/jekyll-sqlite.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "lib/jekyll-sqlite/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "jekyll-sqlite"
+  spec.license = "MIT"
+  spec.version = Jekyll::Sqlite::VERSION
+  spec.authors = ["Nemo"]
+  spec.email = ["jekyll-sqlite@captnemo.in"]
+
+  spec.summary = "A Jekyll plugin to use SQLite databases as a data source."
+  spec.homepage = "https://github.com/captn3m0/jekyll-sqlite"
+  spec.required_ruby_version = ">= 3.0.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "https://github.com/captn3m0/jekyll-sqlite/blob/master/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sqlite3", "~> 2.0"
+  spec.metadata["rubygems_mfa_required"] = "true"
+end

data/lib/jekyll-sqlite/generator.rb

@@ -1,100 +1,140 @@
 # frozen_string_literal: true
+
 require "sqlite3"
 
 module JekyllSQlite
   # Main generator class
   class Generator < Jekyll::Generator
-    safe true
     # Set to high to be higher than the Jekyll Datapages Plugin
     priority :high
-    # Default pragma
-    def fast_setup(db)
-      db.execute("PRAGMA synchronous = OFF")
-      db.execute("PRAGMA journal_mode = OFF")
-      db.execute("PRAGMA query_only = ON")
+
+    ##
+    # Split the given key using dots and return the last part
+    #   customers.order -> order
+    def get_tip(name)
+      name.split(".")[-1]
     end
 
+    ##
     # Get the root of where we are generating the data
     def get_root(root, db_name)
       db_name.split(".")[0..-2].each do |p|
         root = root[p]
+      rescue KeyError
+        raise "Jekyll SQLite: Invalid root. #{p} not found while iterating to #{db_name}"
       end
       root
     end
 
-    def gen_hash_data(root, db, db_name, query)
-      root ||= {}
-
-      root[db_name] = db.execute(query)
-      root[db_name].size
-    end
-
-    def gen_nested_data(item, db, query, db_name)
-      item[db_name] = []
-      db.prepare(query) do |stmt|
-        # We bind params, ignoring any errors
-        # Since there's no way to get required params
-        # From a statement
-        item.each do |key, value|
-          stmt.bind_param key, value
-        rescue StandardError # rubocop:disable Lint/SuppressedException
-        end
-        stmt.execute.each { |d| item[db_name] << d }
+    ##
+    # Prepare the query by binding the parameters
+    # Since we don't know if the query needs them
+    # we ignore all errors about "no such bind parameter"
+    def _prepare_query(stmt, params)
+      params.each do |key, value|
+        stmt.bind_param key, value
+      rescue StandardError => e
+        raise e unless e.message.include? "no such bind parameter"
       end
-      item[db_name].size
     end
 
-    def array_gen(root, config, db_name, db)
-      count = 0
-      root.each do |item|
-        # TODO: Add support for binding Arrays as well.
-        if item.is_a? Hash
-          count += gen_nested_data(item, db, config["query"], db_name)
-        else
-          Jekyll.logger.info "Jekyll SQLite:", "Item is not a hash for #{db_name}. Unsupported configuration"
-        end
+    ##
+    # Internal function to generate data given
+    # root: a Hash-Like root object (site.data, site.data.*, page.data)
+    # key: string as the key to use to attach the data to the root
+    # db: SQLite3 Database object to execute the query on
+    # query: string containing the query to execute
+    # Sets root[db_name] = ResultSet of the query, as an array
+    # Returns the count of the result set
+    def _gen_data(root, key, db, query)
+      db.prepare(query) do |stmt|
+        _prepare_query stmt, get_bind_params(root)
+        root[key] = stmt.execute.to_a
       end
-      count
+      root[key].count
     end
 
-    def gen_data(root, config, db_name, db)
-      count = 0
-      if root.nil? || (root.is_a? Hash)
-        count = gen_hash_data(root, db, db_name, config["query"])
-      elsif root.is_a? Array
-        count = array_gen(root, config, db_name, db)
+    ##
+    # Calls _gen_data for the given root
+    # iterates through the array if root is an array
+    def gen_data(root, ...)
+      if root.is_a? Array
+        # call gen_data for each item in the array
+        # and return the sum of all the counts
+        root.map { |item| gen_data(item, ...) }.sum
+      else
+        _gen_data(root, ...)
       end
-      count
-    end
-
-    def get_tip(name)
-      name.split(".")[-1]
     end
 
+    ##
+    # Validate given configuration object
     def validate_config(config)
       return false unless config.is_a? Hash
       return false unless config.key?("query")
       return false unless File.exist?(config["file"])
       return false unless config.key?("data")
+
       true
     end
 
-    def generate(site)
-      gem_config = site.config['sqlite'] || []
-      gem_config.each do |config|
+    ## pick bindable parameters
+    # from the root
+    # All primitive values are bound to the query
+    # Arrays and Hashes are ignored
+    def get_bind_params(dict)
+      dict.select { |_key, value| !value.is_a?(Array) && !value.is_a?(Hash) }
+    end
+
+    ##
+    # Given a configuration, generate the data
+    # and attach it to the given data_root
+    def generate_data_from_config(root, config)
+      key = config["data"]
+      query = config["query"]
+      file = config["file"]
+      SQLite3::Database.new file, readonly: true do |db|
+        db.results_as_hash = config.fetch("results_as_hash", true)
+
+        branch = get_root(root, key)
+        tip = get_tip(config["data"])
+
+        count = gen_data(branch, tip, db, query)
+        Jekyll.logger.info "Jekyll SQLite:", "Loaded #{key}. Count=#{count}"
+      end
+    end
+
+    ##
+    # Iterate through all the pages in the site
+    # and generate the data from the configuration
+    def gen_pages(site)
+      site.pages.each do |page|
+        gen(page.data, page)
+      end
+    end
+
+    ##
+    # Generate the data from the configuration
+    # Takes as input the root where the data will be attached
+    # and a configuration holder, where the sqlite key can be found
+    # Root is either site.data or page.data
+    # and config_holder is either site.config or page itself.
+    def gen(root, config_holder)
+      sqlite_configs = config_holder["sqlite"] || []
+      sqlite_configs.each do |config|
         unless validate_config(config)
           Jekyll.logger.error "Jekyll SQLite:", "Invalid Configuration. Skipping"
           next
         end
-        d_name = config["data"]
-        SQLite3::Database.new config["file"], readonly: true do |db|
-          fast_setup db
-          db.results_as_hash = config.fetch("results_as_hash", true)
-          root = get_root(site.data, d_name)
-          count = gen_data(root, config, get_tip(d_name), db)
-          Jekyll.logger.info "Jekyll SQLite:", "Loaded #{d_name}. Count=#{count}"
-        end
+        generate_data_from_config(root, config)
       end
     end
+
+    ##
+    # Entrpoint to the generator, called by Jekyll
+    def generate(site)
+      gen(site.data, site.config)
+      gen_pages(site)
+    end
   end
 end

data/lib/jekyll-sqlite/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module Sqlite
-    VERSION = "0.1.3"
+    VERSION = "0.1.5"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-sqlite
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Nemo
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-07-02 00:00:00.000000000 Z
+date: 2024-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -16,15 +16,15 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
-description:
+        version: '2.0'
+description: 
 email:
 - jekyll-sqlite@captnemo.in
 executables: []
@@ -37,6 +37,7 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- jekyll-sqlite.gemspec
 - lib/jekyll-sqlite/generator.rb
 - lib/jekyll-sqlite/version.rb
 - lib/jekyll_sqlite.rb
@@ -48,7 +49,7 @@ metadata:
   source_code_uri: https://github.com/captn3m0/jekyll-sqlite
   changelog_uri: https://github.com/captn3m0/jekyll-sqlite/blob/master/CHANGELOG.md
   rubygems_mfa_required: 'true'
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -63,8 +64,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
-signing_key:
+rubygems_version: 3.5.11
+signing_key: 
 specification_version: 4
 summary: A Jekyll plugin to use SQLite databases as a data source.
 test_files: []