jekyll-sqlite 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25c9ba502f7b75dd59633f9cd88c9412dfe49d0146cd5e4bc3510bd517158c4c
-  data.tar.gz: 29656b5f23cffc2912c498562c5e1547a1e2ffa865cfec3fee35892deaa5d1b5
+  metadata.gz: f3a6e780155f548d59a54d75d24d15d3182c6dedd73345a542a23965585fa6f8
+  data.tar.gz: dbf85f8845cf5190859a4b7ce655ead714e8cdb8422fa605babdde610cd32728
 SHA512:
-  metadata.gz: c0a65da0d781e76cd8bb2605cc57eef3dea1a89e45d059eaf2eb8b9de3b2423b9252831e9d7ae4cc0011097fa9dfc4c533d8593ced72e97e85fe2e86d2151054
-  data.tar.gz: 9c96bdb90d10f05a1d30c3d5e389f180d30e73f69534a047f8e8b755abb16850f4f76a4bdaf3fdf867a71f77a6d93e826a5caac363bca679b6887bdd0160650f
+  metadata.gz: '04964af581a1111c2218572442300402bea84e53350c5ae89204d9d3c4f9eea33b1a4a202d17fe4be61f41ff81421355430599fcdf1d6f3ed642f7f90ffbf048'
+  data.tar.gz: 71a1609f7c901723e205e96864d25f7c2c65bae0492f039ebd9066c1c4ae13ae9044a7ad50a1b636283beb3cd72aa4387e0774ad73fc36e5673dd67a721867cf

data/.rubocop.yml

@@ -2,7 +2,12 @@ require: rubocop-rake
 AllCops:
   TargetRubyVersion: 3.0
   NewCops: enable
-
+  Exclude:
+    - "node_modules/**/*"
+    - "tmp/**/*"
+    - "vendor/**/*"
+    - ".git/**/*"
+    - "test/_plugins/jekyll_sqlite_generator.rb"
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes
@@ -15,4 +20,4 @@ Layout/LineLength:
   Max: 120
 
 Metrics/AbcSize:
-  Max: 20
+  Max: 15

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.1.4] - 2024-07-17
+- Per-page queries are now supported via a `sqlite` config block in the front matter.
+- Documents support for existing site data being used within queries.
+- Code cleanup and refactoring.
+- Don't disable journaling on the database.
+
+## [0.1.3] - 2024-07-02
+- First functional version
+- Adds tests
+
 ## [0.1.0] - 2023-05-08
 
 - Initial release

data/Gemfile

@@ -5,6 +5,13 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in jekyll-sqlite.gemspec
 gemspec
 
+# These are development dependencies
+gem "jekyll", "~> 4.0"
 gem "rake", "~> 13.0"
 gem "rubocop", "~> 1.21"
 gem "rubocop-rake", "~> 0.6.0"
+
+# Ruby 3.4 preparedness
+gem "base64", "~> 0.2.0"
+gem "bigdecimal", "~> 3.1"
+gem "csv", "~> 3.3"

data/README.md

@@ -2,7 +2,8 @@
 
 A Jekyll generator plugin to lets you use SQLite database instead of data files as a data source. It lets you easily create APIs and websites from a SQLite database, by linking together a database file, your template, and the relevant queries.
 
-It additionally supports nested queries, so that you can use the rows of `site.data.items` as bind_params for your nested query.
+It supports site-level queries, per-page queries, and prepared queries that can
+use existing data (possibly generated via more queries) as parameters.
 
 [![Continuous Integration](https://github.com/captn3m0/jekyll-sqlite/actions/workflows/main.yml/badge.svg)](https://github.com/captn3m0/jekyll-sqlite/actions/workflows/main.yml) [![Gem Version](https://badge.fury.io/rb/jekyll-sqlite.svg)](https://badge.fury.io/rb/jekyll-sqlite)
 
@@ -25,88 +26,44 @@ plugins:
 
 ## Usage
 
-Update your `_config.yml` to define your data sources with your SQLite database.
+Update your `_config.yml` to define your data sources with your SQLite database. Please see
+the `test` directory for a functional example with the [Northwind database](https://github.com/jpwhite3/northwind-SQLite3).
 
 ```yml
 ...
-# These are run in sequence, so any nested data can work well.
 sqlite:
-  - data: members
-    file: _db/users.db
-    query: SELECT * FROM members ORDER by created_at DESC
-  # You can use `results_as_hash` to switch between array or hash results (default).
-  - data: verified
-    results_as_hash: false # default true
-    file: _db/users.db
-    query: SELECT username, email FROM members WHERE verified=1
-  - data: members.posts
-    file: _db/posts.db
-    query: SELECT * FROM posts WHERE user_id = :id
+  - data: customers
+    file: *db
+    query: SELECT * from Customers
 ```
 
 Then, you can use the `site.data` attributes accordingly:
 
 ```liquid
-{% for member in site.data.members %}
-- {{member.username}}
-
-  # Your Posts
-  {% for post in member.posts %}
-    {{post}}
-  {% endfor %}
-{% endfor %}
-
-# Result here is an array instead of a hash.
-{% for user in site.data.verified %}
-- :check: {{user[0]}} (Email: {{user[1]}})
-{% endfor %}
+{{ site.data.customers | jsonify }}
 ```
 
-## Generating Pages
-
-It works well with the `datapage_gen` plugin:
+## Prepared Queries
 
-See the [datapage_gen](https://github.com/avillafiorita/jekyll-datapage_gen) docs for more details.
+This plugin supports prepared queries with parameter binding. This lets you
+use existing data from a previous query, or some other source (such as
+`site.data.*` or `page.*`) as a parameter in your query.
 
-Here's a sample configuration:
-
-```yaml
-# This will automatically generate a file for each restaurant
-# restaurants/#{id}.html file
-# with the layout `_layouts/restaurant.html`
-# and page.id, page.name, page.active set
-# and page.title set to restaurant name
-sqlite:
-  restaurants:
-    file: _db/reviews.db
-    sql: SELECT id, name, last_review_date > 1672531200 as active, address FROM restaurants;
-page_gen:
-  - data: restaurants
-    template: restaurant
-    name: id
-    title: name
-    filter: active
-```
-
-## Nested Queries
-
-You can use the rows of `site.data.items` as bind_params for your nested query. For this to work against
-data generated by the plugin, the configuration order must be correct, so you need `site.data.items` above `site.data.items.nested` in your configuration.
-
-Say you have a YAML file defining your items (`data/items.yaml`):
+Say you have a YAML file defining your items (`data/books.yaml`):
 
 ```yaml
 - id: 31323952-2708-42dc-a995-6006a23cbf00
-  name: Item 1
+  name: Time Travel with a Rubber Band
 - id: 5c8e67a0-d490-4743-b5b8-8e67bd1f95a2
-  name: Item 2
+  name: The Art of Cache Invalidation
 ```
 and the prices for the items in your SQLite database, the following configuration will enrich the `items` array with the price:
 
 ```yaml
 sql:
-- data: site.data.items.meta
-  query: SELECT price,author FROM pricing WHERE id =:id
+- data: items.books
+  query: SELECT price, author FROM pricing WHERE id =:id
+  db: books.db
 ```
 This would allow the following Liquid loop to be written:
 
@@ -116,23 +73,62 @@ This would allow the following Liquid loop to be written:
 {% endfor %}
 ```
 
-This works well with `results_as_configuration` as well.
+## Per Page Queries
+
+The exact same syntax can be used on a per-page basis to generate data within
+each page. This is helpful for keeping page-specific queries within the page
+itself. Here's an example:
 
 ```yaml
-sql:
-- data: site.data.items.meta
-  query: SELECT price,author FROM pricing WHERE id =:id
-  results_as_hash: false
+---
+FeaturedSupplierID: 2
+sqlite:
+  - data: suppliers
+    file: "_db/northwind.db"
+    query: "SELECT CompanyName, SupplierID FROM suppliers ORDER BY SupplierID"
+  - data: suppliers.products
+    # This is a prepared query, where SupplierID is coming from the previous query.
+    file: "_db/northwind.db"
+    query: "SELECT ProductName, CategoryID,UnitPrice FROM products WHERE SupplierID = :SupplierID"
+  # :FeaturedSupplierID is picked up automatically from the page frontmatter.
+  - data: FeaturedSupplier
+    file: "_db/northwind.db"
+    query: "SELECT * SupplierID = :FeaturedSupplierID"
+---
+{{page.suppliers|jsonify}}
 ```
 
-The following also renders the price and author:
+This will generate a `page.suppliers` array with all the suppliers, and a `page.FeaturedSupplier` object with the details of the featured supplier.
 
-```liquid
-{% for item in site.data.items %}
-  {{item.meta[0]}}, {{item.meta[1]}}
-{% endfor %}
+Each supplier will have a `products` array with all the products for that supplier.
+
+## Generating Pages
+
+It works well with the `datapage_gen` plugin:
+
+See the [datapage_gen](https://github.com/avillafiorita/jekyll-datapage_gen) docs for more details.
+
+Here's a sample configuration:
+
+```yaml
+sqlite:
+  restaurants:
+    file: _db/reviews.db
+    sql: SELECT id, name, last_review_date > 1672531200 as active, address FROM restaurants;
+page_gen:
+  - data: restaurants
+    template: restaurant
+    name: id
+    title: name
+    filter: active
 ```
 
+This will automatically generate a file for each restaurant
+restaurants/#{id}.html file with the layout `_layouts/restaurant.html` and page.id, page.name, page.active set and page.title set to restaurant name
+
+Note that the `datapage_gen` plugin will run _after_ the `jekyll-sqlite` plugin,
+if you generate any pages with per-page queries, these queries will not execute.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -2,7 +2,67 @@
 
 require "bundler/gem_tasks"
 require "rubocop/rake_task"
+require "jekyll"
+require "sqlite3"
 
 RuboCop::RakeTask.new
 
+def assert(cond, msg = "Assertion Failed")
+  raise msg unless cond
+end
+
+def query_db(query)
+  db = SQLite3::Database.new "_db/northwind.db"
+  db.results_as_hash = true
+  results = db.execute query
+  results[0]
+end
+
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Metrics/MethodLength
+def validate_json
+  file = "_site/data.json"
+  data = JSON.parse(File.read(file))
+  assert data["orders"].size == 53, "Expected 53 orders, got #{data["orders"].size}"
+  assert data["customers"].size == 93, "Expected 93 customers, got #{data["customers"].size}"
+  assert data["categories"].size == 8, "Expected 93 categories, got #{data["categories"].size}"
+  assert data["orders"][0] == query_db("SELECT * FROM Orders LIMIT 1"), "Order Fetch Failed"
+  assert data["customers"][0] == query_db("SELECT * FROM Customers LIMIT 1"), "Customer Fetch Failed"
+  assert data["customers"][0] == query_db("SELECT * FROM Customers LIMIT 1"), "Customer Fetch Failed"
+  assert data["categories"][0]["products"].size == 12, "Products don't match"
+  data["categories"][0]["products"].each do |p|
+    assert p["CategoryID"] == 1, "CategoryID doesn't match"
+  end
+  assert data["delayedOrders"].size == 17
+  assert data["delayedOrders"][0]["OrderID"] == 10_249
+end
+
+def validate_page_json
+  file = "_site/suppliers.json"
+  read_data = JSON.parse(File.read(file))
+  data = read_data["allSuppliers"]
+  assert data.size == 29, "Expected 29 suppliers, got #{data.size}"
+  r = query_db("SELECT CompanyName, SupplierID FROM Suppliers ORDER BY SupplierID LIMIT 1")
+  assert r["CompanyName"] == data[0]["CompanyName"], "Company Name doesn't match"
+  assert r["SupplierID"] == data[0]["SupplierID"], "Supplier ID doesn't match"
+  assert data[0]["products"].size == 3, "Products don't match"
+  assert data[0]["products"][0]["ProductName"] == "Chai"
+  assert data[0]["products"][1]["ProductName"] == "Chang"
+  assert data[0]["products"][2]["ProductName"] == "Aniseed Syrup"
+
+  # Focus Supplier - this uses the data from the page front-matter to prepare a query
+  fs = query_db("SELECT * FROM Suppliers WHERE SupplierID = 6")
+  assert read_data["focusSupplier"][0] == fs, "Focus Supplier doesn't match"
+end
+# rubocop:enable Metrics/AbcSize
+# rubocop:enable Metrics/MethodLength
+
 task default: :rubocop
+
+desc "Build Test Site"
+task :test do
+  Dir.chdir("test")
+  Jekyll::Site.new(Jekyll.configuration).process
+  validate_json
+  validate_page_json
+end

data/jekyll-sqlite.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "lib/jekyll-sqlite/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "jekyll-sqlite"
+  spec.license = "MIT"
+  spec.version = Jekyll::Sqlite::VERSION
+  spec.authors = ["Nemo"]
+  spec.email = ["jekyll-sqlite@captnemo.in"]
+
+  spec.summary = "A Jekyll plugin to use SQLite databases as a data source."
+  spec.homepage = "https://github.com/captn3m0/jekyll-sqlite"
+  spec.required_ruby_version = ">= 3.0.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "https://github.com/captn3m0/jekyll-sqlite/blob/master/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sqlite3", "~> 1.6"
+  spec.metadata["rubygems_mfa_required"] = "true"
+end

data/lib/jekyll-sqlite/generator.rb

@@ -1,86 +1,140 @@
 # frozen_string_literal: true
-require 'sqlite3'
+
+require "sqlite3"
+
 module JekyllSQlite
   # Main generator class
   class Generator < Jekyll::Generator
-    safe true
     # Set to high to be higher than the Jekyll Datapages Plugin
     priority :high
-    # Default pragma
-    def fast_setup(db)
-      db.execute("PRAGMA synchronous = OFF")
-      db.execute("PRAGMA journal_mode = OFF")
-      db.execute("PRAGMA query_only = ON")
+
+    ##
+    # Split the given key using dots and return the last part
+    #   customers.order -> order
+    def get_tip(name)
+      name.split(".")[-1]
     end
 
+    ##
     # Get the root of where we are generating the data
-    def get_root(root, name)
-      name.split(".")[0..-2].each do |p|
+    def get_root(root, db_name)
+      db_name.split(".")[0..-2].each do |p|
         root = root[p]
+      rescue KeyError
+        raise "Jekyll SQLite: Invalid root. #{p} not found while iterating to #{db_name}"
       end
       root
     end
 
-    def gen_hash_data(root, db, _query)
-      root ||= {}
-
-      root[name] = db.execute(config["query"])
-      root[name].size
+    ##
+    # Prepare the query by binding the parameters
+    # Since we don't know if the query needs them
+    # we ignore all errors about "no such bind parameter"
+    def _prepare_query(stmt, params)
+      params.each do |key, value|
+        stmt.bind_param key, value
+      rescue StandardError => e
+        raise e unless e.message.include? "no such bind parameter"
+      end
     end
 
-    def gen_nested_data(item, db, query, name)
-      item[name] = []
+    ##
+    # Internal function to generate data given
+    # root: a Hash-Like root object (site.data, site.data.*, page.data)
+    # key: string as the key to use to attach the data to the root
+    # db: SQLite3 Database object to execute the query on
+    # query: string containing the query to execute
+    # Sets root[db_name] = ResultSet of the query, as an array
+    # Returns the count of the result set
+    def _gen_data(root, key, db, query)
       db.prepare(query) do |stmt|
-        # We bind params, ignoring any errors
-        # Since there's no way to get required params
-        # From a statement
-        item.each do |key, value|
-          stmt.bind_param key, value
-        rescue StandardError # rubocop:disable Lint/SuppressedException
-        end
-        stmt.execute.each { |d| item[name] << d }
+        _prepare_query stmt, get_bind_params(root)
+        root[key] = stmt.execute.to_a
       end
-      item[name].size
+      root[key].count
     end
 
-    def array_gen(root, config, name, db)
-      count = 0
-      root.each do |item|
-        # TODO: Add support for binding Arrays as well.
-        if item.is_a? Hash
-          count += gen_nested_data(item, db, config["query"], name)
-        else
-          Jekyll.logger.info "Jekyll SQLite:", "Item is not a hash for #{name}. Unsupported configuration"
-        end
+    ##
+    # Calls _gen_data for the given root
+    # iterates through the array if root is an array
+    def gen_data(root, ...)
+      if root.is_a? Array
+        # call gen_data for each item in the array
+        # and return the sum of all the counts
+        root.map { |item| gen_data(item, ...) }.sum
+      else
+        _gen_data(root, ...)
       end
-      count
     end
 
-    def gen_data(root, config, name, db)
-      count = 0
-      if root.nil? || (root.is_a? Hash)
-        count = gen_hash_data(root, db, config["query"])
-      elsif root.is_a? Array
-        count = array_gen(root, config, name, db)
+    ##
+    # Validate given configuration object
+    def validate_config(config)
+      return false unless config.is_a? Hash
+      return false unless config.key?("query")
+      return false unless File.exist?(config["file"])
+      return false unless config.key?("data")
+
+      true
+    end
+
+    ## pick bindable parameters
+    # from the root
+    # All primitive values are bound to the query
+    # Arrays and Hashes are ignored
+    def get_bind_params(dict)
+      dict.select { |_key, value| !value.is_a?(Array) && !value.is_a?(Hash) }
+    end
+
+    ##
+    # Given a configuration, generate the data
+    # and attach it to the given data_root
+    def generate_data_from_config(root, config)
+      key = config["data"]
+      query = config["query"]
+      file = config["file"]
+      SQLite3::Database.new file, readonly: true do |db|
+        db.results_as_hash = config.fetch("results_as_hash", true)
+
+        branch = get_root(root, key)
+        tip = get_tip(config["data"])
+
+        count = gen_data(branch, tip, db, query)
+        Jekyll.logger.info "Jekyll SQLite:", "Loaded #{key}. Count=#{count}"
       end
-      count
     end
 
-    def get_tip(name)
-      name.split(".")[-1]
+    ##
+    # Iterate through all the pages in the site
+    # and generate the data from the configuration
+    def gen_pages(site)
+      site.pages.each do |page|
+        gen(page.data, page)
+      end
     end
 
-    def generate(site)
-      site.config["sqlite"].each do |config|
-        name = config["data"]
-        SQLite3::Database.new config["file"], readonly: true do |db|
-          fast_setup db
-          db.results_as_hash = config.fetch("results_as_hash", true)
-          root = get_root(site.data, name)
-          count = gen_data(root, config, get_tip(name), db)
-          Jekyll.logger.info "Jekyll SQLite:", "Loaded #{name}. Count=#{count}. as_hash=#{db.results_as_hash}"
+    ##
+    # Generate the data from the configuration
+    # Takes as input the root where the data will be attached
+    # and a configuration holder, where the sqlite key can be found
+    # Root is either site.data or page.data
+    # and config_holder is either site.config or page itself.
+    def gen(root, config_holder)
+      sqlite_configs = config_holder["sqlite"] || []
+      sqlite_configs.each do |config|
+        unless validate_config(config)
+          Jekyll.logger.error "Jekyll SQLite:", "Invalid Configuration. Skipping"
+          next
         end
+        generate_data_from_config(root, config)
       end
     end
+
+    ##
+    # Entrpoint to the generator, called by Jekyll
+    def generate(site)
+      gen(site.data, site.config)
+      gen_pages(site)
+    end
   end
 end

data/lib/jekyll-sqlite/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module Sqlite
-    VERSION = "0.1.2"
+    VERSION = "0.1.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-sqlite
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Nemo
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-05-11 00:00:00.000000000 Z
+date: 2024-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -24,9 +24,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-description:
+description: 
 email:
-- jekyll@captnemo.in
+- jekyll-sqlite@captnemo.in
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -35,9 +35,9 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - README.md
 - Rakefile
+- jekyll-sqlite.gemspec
 - lib/jekyll-sqlite/generator.rb
 - lib/jekyll-sqlite/version.rb
 - lib/jekyll_sqlite.rb
@@ -49,7 +49,7 @@ metadata:
   source_code_uri: https://github.com/captn3m0/jekyll-sqlite
   changelog_uri: https://github.com/captn3m0/jekyll-sqlite/blob/master/CHANGELOG.md
   rubygems_mfa_required: 'true'
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -64,9 +64,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.25
-signing_key:
+rubygems_version: 3.5.11
+signing_key: 
 specification_version: 4
-summary: A Jekyll plugin that lets you use SQLite database instead of data files as
-  a data source.
+summary: A Jekyll plugin to use SQLite databases as a data source.
 test_files: []

data/Gemfile.lock

@@ -1,49 +0,0 @@
-PATH
-  remote: .
-  specs:
-    jekyll-sqlite (0.1.1)
-      sqlite3 (~> 1.6)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.4.2)
-    json (2.6.3)
-    parallel (1.23.0)
-    parser (3.2.2.1)
-      ast (~> 2.4.1)
-    rainbow (3.1.1)
-    rake (13.0.6)
-    regexp_parser (2.8.0)
-    rexml (3.2.5)
-    rubocop (1.50.2)
-      json (~> 2.3)
-      parallel (~> 1.10)
-      parser (>= 3.2.0.0)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.0, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.1)
-      parser (>= 3.2.1.0)
-    rubocop-rake (0.6.0)
-      rubocop (~> 1.0)
-    ruby-progressbar (1.13.0)
-    sqlite3 (1.6.2-aarch64-linux)
-    sqlite3 (1.6.2-x86_64-linux)
-    unicode-display_width (2.4.2)
-
-PLATFORMS
-  aarch64-linux
-  x86_64-linux
-
-DEPENDENCIES
-  jekyll-sqlite!
-  rake (~> 13.0)
-  rubocop (~> 1.21)
-  rubocop-rake (~> 0.6.0)
-
-BUNDLED WITH
-   2.4.1