jekyll-generate-tags 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: db89f3d933db9c98e8d73fd9614cbeb3448c7efdb944ffb8408c51457ec13d47
+  data.tar.gz: ffe7e155720e1c28ce180065b32a6091f80e5d2a8920c9e4195bc6575b69ed0a
+SHA512:
+  metadata.gz: 6d624f2eef4068bd51ae550e9c7ccab7a2e8dc5a3a94b467a20389671bc81cbaea84c8aaee46c7b37f03d3b29ea63f2e15bf950857780172e6cb55fb5f38bff5
+  data.tar.gz: a90a9dee0fae9a44a9942109c2908aaeee9a9b50865cbc0271a26a4daf976493f332c8e216d92ea086e8e6511d93d9f8b18b58f15bb3233917df2ce208634f59

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+/vendor/
+Gemfile.lock
+.tags-cache

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in jekyll-generate-tags.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Pat Migliaccio pat@patmigliaccio.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+# jekyll-generate-tags
+
+Classify Jekyll post content automatically using NLP ([huggingface.co/facebook/bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli)) based on a set of site-wide labels.
+
+## Installation
+
+### Prerequisites
+
+* Python
+
+### Install
+
+Add this line to your project's Gemfile:
+
+```ruby
+gem 'jekyll-generate-tags'
+```
+
+Add the following to your site's _config.yml:
+
+```yml
+plugins:
+  - jekyll-generate-tags
+```
+
+## Usage
+
+Use the following tag to reference a comma-delimited string of tag matches (in order of confidence).
+
+```md           
+{{ page.generatedtags }}
+```
+
+_Note: When building the site for the first time, it may take a bit to run through existing posts to generate each set of tags._
+
+### Custom Labels
+
+_Optional_: Add the following property to the `_config.yml` file to include custom labels for selection as tags.
+
+```yml
+tag_options: software engineering,coding,programming languages
+```
+
+These can also be updated in the `tag_options.txt` file within the root of the project or updated inside the gem itself.
+
+### Confidence
+
+By default, labels are only selected as tags if they have a greater than `.50` confidence value.
+
+_Optional_: Add the following property to the `_config.yml` file to alter the confidence value according to desired preference.
+
+```yml
+tag_confidence: .30
+```
+
+## Caching
+
+Includes a caching-mechanism that allows for faster re-builds when the content, labels, or confidence values don't change. The outputted tags are stored in the `.tags-cache` folder. Depending on how your site builds are handled, it probably makes sense to check this folder in to eliminate the need to reprocess each post at time of deployment.
+
+### Clear Cache
+
+To clear the cache entirely, simply delete the `.tags-cache` folder.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "jekyll/generate/tags"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+bundle exec rake build

data/jekyll-generate-tags.gemspec

@@ -0,0 +1,30 @@
+require_relative 'lib/jekyll-generate-tags/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "jekyll-generate-tags"
+  spec.version       = Jekyll::GenerateTags::VERSION
+  spec.authors       = ["Pat Migliaccio"]
+  spec.email         = ["pat@patmigliaccio.com"]
+
+  spec.summary       = "Classify Jekyll post content automatically using NLP"
+  spec.homepage      = "https://github.com/patmigliaccio/jekyll-generate-tags"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/patmigliaccio/jekyll-generate-tags"
+  spec.metadata["changelog_uri"] = "https://github.com/patmigliaccio/jekyll-generate-tags"
+
+  spec.add_runtime_dependency "jekyll", "~> 3.9.2"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/lib/jekyll-generate-tags/generate.py

@@ -0,0 +1,18 @@
+import sys
+from transformers import pipeline
+
+content = sys.argv[1]
+tags = sys.argv[2]
+confidence = float(sys.argv[3])
+
+classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
+
+data = classifier(content, tags)
+
+labels = data['labels']
+scores = data['scores']
+
+# Filter the labels based on the score
+filtered_labels = [label for label, score in zip(labels, scores) if score > confidence]
+
+print(",".join(filtered_labels))

data/lib/jekyll-generate-tags/generator.rb

@@ -0,0 +1,90 @@
+require 'shellwords'
+require 'openssl'
+require 'fileutils'
+
+require "jekyll"
+require "jekyll-generate-tags/version"
+
+module Jekyll
+  module GenerateTags
+    class Error < StandardError; end
+
+    class Generator
+      @@hash_key = "cache-key"
+      @@cache_folder_name = ".tags-cache"
+      @@tag_options_name = "tag_options.txt"
+      @@default_confidence = ".50"
+
+      def generate_tags(content, tags, confidence)
+        # Do tags exist in `_config.yml`?
+        if tags.nil?
+          # Look for tags in project root
+          tags_path = File.expand_path(@@tag_options_name)
+          if !File.exists?(tags_path)
+            # Use default tags from gem
+            tags_path = File.expand_path(@@tag_options_name, __dir__)
+          end
+          tags_file = File.read(tags_path)
+          lines = tags_file.split("\n")
+          tags = lines.join(",")
+        end
+
+        if confidence.nil?
+          confidence = @@default_confidence
+        end
+
+        cache_key = content + tags + String(confidence)
+
+        result = self.get_cache(cache_key)
+        if result.nil?
+          script_path = File.expand_path("generate.py", __dir__)
+          arg1 = Shellwords.escape(content)
+          arg2 = Shellwords.escape(tags)
+          arg3 = Shellwords.escape(String(confidence))
+          result = `python #{script_path} #{arg1} #{arg2} #{arg3}`
+
+          self.set_cache(cache_key, result)
+        end
+
+        result
+      end
+
+      # Stores newly generated tags in a file for fast reuse
+      def set_cache(key, value)
+        cache_path = get_cache_file_path(key)
+        # Store file
+        File.open(cache_path, "w") do |file|
+          file.puts(value)
+        end
+      end
+
+      # Retrieves existing generated tags from a cache file for fast reuse
+      def get_cache(key)
+        cache_path = get_cache_file_path(key)
+        # Retrieve value from file
+        if File.exists?(cache_path)
+          result = File.read(cache_path)
+          result
+        end
+      end
+
+      def create_cache_folder()
+        cache_folder_path = File.expand_path(@@cache_folder_name)
+        # Check if the folder exists
+        if !Dir.exists?(cache_folder_path)
+          # Create the folder
+          FileUtils.mkdir_p(cache_folder_path)
+        end
+      end
+
+      def get_cache_file_path(key)
+        create_cache_folder()
+
+        # Create the HMAC
+        hmac = OpenSSL::HMAC.hexdigest('sha256', @@hash_key, key)
+        cache_path = File.expand_path("#{@@cache_folder_name}/#{hmac}.txt")
+        cache_path
+      end
+    end
+  end
+end

data/lib/jekyll-generate-tags/installer.rb

@@ -0,0 +1,10 @@
+require "rubygems/installer"
+
+module Jekyll
+  class Installer < Gem::Installer
+    def install
+      # install the Python packages from the requirements.txt file
+      system "pip install -r #{File.join(gem_dir, "requirements.txt")}"
+    end
+  end
+end

data/lib/jekyll-generate-tags/requirements.txt

@@ -0,0 +1,4 @@
+transformers>=4.25.1
+torch>=1.13.1
+torchvision>=0.14.1
+sentencepiece>=0.1.97

data/lib/jekyll-generate-tags/setup.py

@@ -0,0 +1,6 @@
+from setuptools import setup
+
+setup(
+    name='jekyll-generate-tags',
+    version='0.1',
+)

data/lib/jekyll-generate-tags/version.rb

@@ -0,0 +1,5 @@
+module Jekyll
+  module GenerateTags
+    VERSION = "0.1.0"
+  end
+end

data/lib/jekyll-generate-tags.rb

@@ -0,0 +1,22 @@
+# frozen-string-literal: true
+
+require "jekyll"
+require "jekyll-generate-tags/generator"
+
+# register the generate_tags method as a Jekyll hook
+Jekyll::Hooks.register :site, :pre_render do |site, payload|
+  generator = Jekyll::GenerateTags::Generator.new
+
+  tag_options = site.config["tag_options"]
+  tag_confidence = site.config["tag_confidence"]
+
+  posts = payload["site"]["posts"]
+  posts.each do |post|
+    post_name = File.basename(post.path, '.md')
+
+    tags = generator.generate_tags(post.content, tag_options, tag_confidence)
+  
+    # add the generated tags to the document's front matter
+    post.data["generatedtags"] = tags
+  end
+end

data/tag_options.txt

@@ -0,0 +1,3 @@
+software engineering
+coding
+programming languages

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: jekyll-generate-tags
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Pat Migliaccio
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-01-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jekyll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.9.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.9.2
+description:
+email:
+- pat@patmigliaccio.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- jekyll-generate-tags.gemspec
+- lib/jekyll-generate-tags.rb
+- lib/jekyll-generate-tags/generate.py
+- lib/jekyll-generate-tags/generator.rb
+- lib/jekyll-generate-tags/installer.rb
+- lib/jekyll-generate-tags/requirements.txt
+- lib/jekyll-generate-tags/setup.py
+- lib/jekyll-generate-tags/version.rb
+- tag_options.txt
+homepage: https://github.com/patmigliaccio/jekyll-generate-tags
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/patmigliaccio/jekyll-generate-tags
+  source_code_uri: https://github.com/patmigliaccio/jekyll-generate-tags
+  changelog_uri: https://github.com/patmigliaccio/jekyll-generate-tags
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: Classify Jekyll post content automatically using NLP
+test_files: []