appydave-tools 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14a6bd2963b58f85f247282a6cf8c537495b9a18300b04c3a3794ef5df4ecaa1
-  data.tar.gz: 5ec47924ab4268377a297e1f02f6fa6b4f62207179f21fe8ccdcd69d62cd4f8d
+  metadata.gz: e4f0fbf0d8d8efc742e691efe1e128bb49f5a1f1423fe2b5eb7a700d9d606c24
+  data.tar.gz: 4a6ea3bba78bbb0c8983f462888e0c7e8c1910706e90bbb9db0bfbad99e47405
 SHA512:
-  metadata.gz: 253546f33096bad9f9daaa5776fa9d98142a4e7cb43607bf2040ab648d49f83712c8f491b8730de4bd602bcd4080a9360cb9bdd082a1b249df4210d66083ebf6
-  data.tar.gz: f1f638a4a299df238694c8b4e42c008e8663a08cb2b83931a953796ec6679fa83038bebe3f749b0351a7edd848220cb906c53ff076abbf953564ff0356347b29
+  metadata.gz: 85058aef1eccdf67319578d1d4c929a17da0143492ba91095ab7e09702f12648174a17bcb611ab5e2951ab500386b98a7f020c5695772c488ebd782bc275e3a6
+  data.tar.gz: fb4b852d1dff53cb256c190e586546d5ee2bf13a675da10b0d25c6abc4d04588d541541763bff650609816677e7e3fe407c9920508dcd843a8ab19ac6c2943ab

data/.rubocop.yml

@@ -45,6 +45,9 @@ RSpec/ExampleLength:
 
 Metrics/MethodLength:
   Max: 25
+  Exclude:
+    - "**/spec/**/*"
+    - "bin/*.rb"
 
 Layout/LineLength:
   Max: 200
@@ -121,12 +124,15 @@ Metrics/AbcSize:
   Max: 25
   Exclude:
     - "bin/*"
+    - "lib/appydave/tools/subtitle_master/clean.rb"
 Metrics/CyclomaticComplexity:
   Exclude:
     - "**/models/**/*"
     - "lib/appydave/tools/bank_reconciliation/clean/mapper.rb"
+    - "lib/appydave/tools/subtitle_master/clean.rb"
 Metrics/PerceivedComplexity:
   Exclude:
     - "**/models/**/*"
+    - "lib/appydave/tools/subtitle_master/clean.rb"
 RSpec/MultipleMemoizedHelpers:
   Enabled: false

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [0.7.0](https://github.com/klueless-io/appydave-tools/compare/v0.6.1...v0.7.0) (2024-05-29)
+
+
+### Features
+
+* new tool for doing bank reconciliations with chart of account matching ([9b82605](https://github.com/klueless-io/appydave-tools/commit/9b8260571f6046470d5963354ee1c80e493a0f28))
+
 ## [0.6.1](https://github.com/klueless-io/appydave-tools/compare/v0.6.0...v0.6.1) (2024-05-26)
 
 

data/bin/bank_reconciliation.rb

@@ -32,19 +32,42 @@ class BankReconciliationCLI
   private
 
   def clean_transactions(args)
-    options = {}
+    options = { include: [] }
     OptionParser.new do |opts|
       opts.banner = 'Usage: bank_reconciliation.rb clean [options]'
-      opts.on('-i', '--include PATTERN', 'GLOB pattern for source transaction files') { |v| options[:include] = v }
-      opts.on('-f', '--transaction FOLDER', 'Transaction CSV folder where original banking CSV files are stored') { |v| options[:transaction_folder] = v }
-      opts.on('-o', '--output FILE', 'Output CSV file name') { |v| options[:output] = v }
+
+      opts.on('-i', '--include PATTERN', 'GLOB pattern for source transaction files') do |v|
+        options[:include] << v
+      end
+
+      opts.on('-f', '--transaction FOLDER', 'Transaction CSV folder where original banking CSV files are stored') do |v|
+        options[:transaction_folder] = v
+      end
+
+      opts.on('-o', '--output FILE', 'Output CSV file name') do |v|
+        options[:output] = v
+      end
+
       opts.on_tail('-h', '--help', 'Show this message') do
         puts opts
         exit
       end
     end.parse!(args)
 
-    # Implement cleaning and normalizing transactions
+    transaction_folder = options[:transaction_folder] || '/default/transaction/folder'
+    output_file = options[:output] || 'clean_transactions.csv'
+    include_patterns = options[:include].empty? ? ['*'] : options[:include]
+
+    puts "Cleaning transactions with options: #{options}"
+
+    # Ensure the clean directory exists
+    clean_dir = File.dirname(output_file)
+    FileUtils.mkdir_p(clean_dir)
+
+    # Initialize the CleanTransactions class and process the files
+    cleaner = Appydave::Tools::BankReconciliation::Clean::CleanTransactions.new(transaction_folder: transaction_folder)
+    cleaner.clean_transactions(include_patterns, output_file)
+
     puts "Cleaning transactions with options: #{options}"
   end
 
@@ -95,5 +118,7 @@ class BankReconciliationCLI
   end
 end
 
+Appydave::Tools::Configuration::Config.configure
+
 BankReconciliationCLI.new.run
 # BankReconciliationCLI.new.run if __FILE__ == $PROGRAM_NAME

data/bin/subtitle_master.rb

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+
+require 'appydave/tools'
+
+# Process command line arguments for SubtitleMaster operations
+class SubtitleMasterCLI
+  def initialize
+    @commands = {
+      'clean' => method(:clean_subtitles),
+      'correct' => method(:correct_subtitles),
+      'split' => method(:split_subtitles),
+      'highlight' => method(:highlight_subtitles),
+      'image_prompts' => method(:generate_image_prompts)
+    }
+  end
+
+  def run
+    command, *args = ARGV
+    if @commands.key?(command)
+      @commands[command].call(args)
+    else
+      puts "Unknown command: #{command}"
+      print_help
+    end
+  end
+
+  private
+
+  def clean_subtitles(args)
+    options = parse_options(args, 'clean')
+    cleaner = Appydave::Tools::SubtitleMaster::Clean.new(options[:file])
+    result = cleaner.clean
+    write_output(result, options[:output])
+  end
+
+  def correct_subtitles(args)
+    options = parse_options(args, 'correct')
+    corrector = Appydave::Tools::SubtitleMaster::Correct.new(options[:file])
+    result = corrector.correct
+    write_output(result, options[:output])
+  end
+
+  def split_subtitles(args)
+    options = parse_options(args, 'split', %i[words_per_group])
+    splitter = Appydave::Tools::SubtitleMaster::Split.new(options[:file], options[:words_per_group])
+    result = splitter.split
+    write_output(result, options[:output])
+  end
+
+  def highlight_subtitles(args)
+    options = parse_options(args, 'highlight')
+    highlighter = Appydave::Tools::SubtitleMaster::Highlight.new(options[:file])
+    result = highlighter.highlight
+    write_output(result, options[:output])
+  end
+
+  def generate_image_prompts(args)
+    options = parse_options(args, 'image_prompts')
+    image_prompter = Appydave::Tools::SubtitleMaster::ImagePrompts.new(options[:file])
+    result = image_prompter.generate_prompts
+    write_output(result, options[:output])
+  end
+
+  def parse_options(args, command, extra_options = [])
+    options = { file: nil, output: nil }
+    OptionParser.new do |opts|
+      opts.banner = "Usage: subtitle_master.rb #{command} [options]"
+
+      opts.on('-f', '--file FILE', 'SRT file to process') { |v| options[:file] = v }
+      opts.on('-o', '--output FILE', 'Output file') { |v| options[:output] = v }
+
+      extra_options.each do |opt|
+        case opt
+        when :words_per_group
+          opts.on('-w', '--words-per-group WORDS', 'Number of words per group for splitting') { |v| options[:words_per_group] = v.to_i }
+        end
+      end
+
+      opts.on_tail('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end.parse!(args)
+
+    unless options[:file] && options[:output]
+      puts 'Missing required options. Use -h for help.'
+      exit
+    end
+
+    options
+  end
+
+  def write_output(result, output_file)
+    File.write(output_file, result)
+    puts "Processed file written to #{output_file}"
+  end
+
+  def print_help
+    puts 'Usage: subtitle_master.rb [command] [options]'
+    puts 'Commands:'
+    puts '  clean          Clean and normalize SRT files'
+    puts '  correct        Correct common typos and mistranslations in SRT files'
+    puts '  split          Split subtitle groups based on word count'
+    puts '  highlight      Highlight power words in subtitles'
+    puts '  image_prompts  Generate image prompts from subtitle text'
+    puts "Run 'subtitle_master.rb [command] --help' for more information on a command."
+  end
+end
+
+SubtitleMasterCLI.new.run

data/lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb

@@ -10,24 +10,31 @@ module Appydave
           include KLog::Logging
 
           attr_reader :transaction_folder
+          attr_reader :output_folder
           attr_reader :transactions
 
           # (config_file)
-          def initialize(transaction_folder: '/Volumes/Expansion/Sync/bank-reconciliation/original-transactions')
+          def initialize(transaction_folder: nil, output_folder: nil)
+            # needs to use config.bank_reconciliation.transaction_folder
+            transaction_folder ||= '/Volumes/Expansion/Sync/bank-reconciliation/original-transactions'
+            output_folder ||= File.join(transaction_folder, 'clean')
+
             @transaction_folder = transaction_folder
+            @output_folder = output_folder
           end
 
-          def clean_transactions(input_globs, _output_file)
+          def clean_transactions(input_globs, output_file)
             raw_transactions = grab_raw_transactions(input_globs)
             transactions, duplicates_count = deduplicate(raw_transactions)
 
             transactions = Mapper.new.map(transactions)
-            # tp transactions
+
+            # tp transactions, Appydave::Tools::BankReconciliation::Models::Transaction.csv_headers
 
             log.kv 'Deduped consolidated transactions', duplicates_count if duplicates_count.positive?
 
-            # transactions = normalize(transactions)
-            # save_to_csv(transactions, output_file)
+            save_to_csv(transactions, output_file)
+
             @transactions = transactions
           end
 
@@ -42,6 +49,7 @@ module Appydave
 
               input_globs.each do |glob|
                 Dir.glob(glob).each do |file|
+                  log.kv 'Reading transactions from', file
                   raw_transactions = ReadTransactions.new(file).read
                   deduped_transactions, duplicates_count = deduplicate(raw_transactions)
 
@@ -81,8 +89,11 @@ module Appydave
           end
 
           def save_to_csv(transactions, output_file)
+            FileUtils.mkdir_p(output_folder)
+            output_file = File.join(output_folder, output_file)
+
             CSV.open(output_file, 'w') do |csv|
-              csv << ReconciledTransaction.csv_headers
+              csv << Appydave::Tools::BankReconciliation::Models::Transaction.csv_headers
               transactions.each do |transaction|
                 csv << transaction.to_csv_row
               end

data/lib/appydave/tools/bank_reconciliation/clean/mapper.rb

@@ -40,7 +40,9 @@ module Appydave
               trigram_match(transaction, 0.7, '70%') ||
               trigram_match(transaction, 0.6, '60%') ||
               trigram_match(transaction, 0.5, '50%') ||
-              transaction
+              start_with_match(transaction) ||
+              includes(transaction)
+            transaction
           end
 
           def map_bank_account(transaction)
@@ -56,7 +58,7 @@ module Appydave
 
           def equality_match(transaction)
             coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
-              chart_of_account.narration.to_s.delete(' ') == transaction.narration.delete(' ')
+              chart_of_account.narration.to_s.delete(' ').downcase == transaction.narration.delete(' ').downcase
             end
 
             return nil unless coa
@@ -66,6 +68,30 @@ module Appydave
             transaction
           end
 
+          def start_with_match(transaction)
+            coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
+              transaction.narration.to_s.delete(' ').downcase.start_with?(chart_of_account.narration.to_s.downcase)
+            end
+
+            return nil unless coa
+
+            transaction.coa_match_type = 'starts_with'
+            transaction.coa_code = coa.code
+            transaction
+          end
+
+          def includes(transaction)
+            coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
+              transaction.narration.to_s.delete(' ').downcase.include?(chart_of_account.narration.delete(' ').to_s.downcase)
+            end
+
+            return nil unless coa
+
+            transaction.coa_match_type = 'includes'
+            transaction.coa_code = coa.code
+            transaction
+          end
+
           def trigram_match(transaction, score_threshold, match_type)
             scored_transactions = config.bank_reconciliation.chart_of_accounts.map do |coa|
               {

data/lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb

@@ -21,6 +21,8 @@ module Appydave
             case platform
             when :bankwest
               read_bankwest(csv_lines)
+            when :bankwest2
+              read_bankwest2(csv_lines)
             end
           end
 
@@ -48,11 +50,35 @@ module Appydave
             @transactions
           end
 
+          def read_bankwest2(csv_lines)
+            @transactions = []
+
+            # Skip the header line and parse each subsequent line
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              transaction = Models::Transaction.new(
+                bsb_number: row['BSB / Account Number'].split(' - ').first,
+                account_number: row['BSB / Account Number'].split(' - ').last,
+                transaction_date: row['Transaction Date'],
+                narration: row['Narration'],
+                cheque_number: row['Cheque Number'],
+                debit: row['Debit'],
+                credit: row['Credit'],
+                balance: row['Balance'],
+                transaction_type: row['Transaction Type']
+              )
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
           # For bankwest the first row is the CSV will look like:
           # BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type
           def detect_platform(csv_lines)
             return :bankwest if csv_lines.first.start_with?('BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
+            return :bankwest2 if csv_lines.first.start_with?('Account Name,BSB / Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
 
+            puts "Unknown platform detected. CSV columns are: #{csv_lines.first.strip}"
             raise Appydave::Tools::Error, 'Unknown platform'
           end
         end

data/lib/appydave/tools/bank_reconciliation/models/transaction.rb

@@ -48,6 +48,42 @@ module Appydave
             @coa_match_type = coa_match_type
             @account_name = account_name
           end
+
+          def self.csv_headers
+            %i[
+              bsb_number
+              account_number
+              transaction_date
+              narration
+              cheque_number
+              debit
+              credit
+              balance
+              transaction_type
+              platform
+              coa_code
+              coa_match_type
+              account_name
+            ]
+          end
+
+          def to_csv_row
+            [
+              @bsb_number,
+              @account_number,
+              @transaction_date,
+              @narration,
+              @cheque_number,
+              @debit,
+              @credit,
+              @balance,
+              @transaction_type,
+              @platform,
+              @coa_code,
+              @coa_match_type,
+              @account_name
+            ]
+          end
         end
       end
     end

data/lib/appydave/tools/subtitle_master/_doc.md

@@ -0,0 +1,3 @@
+# Subtitle Master
+
+[ChatGPT](https://chatgpt.com/c/f80dfca5-8168-4561-b5c6-8efed8672a88)

data/lib/appydave/tools/subtitle_master/clean.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Appydave
+  module Tools
+    module SubtitleMaster
+      # Clean and normalize subtitles
+      class Clean
+        def initialize(file_path)
+          @file_path = file_path
+        end
+
+        def clean
+          content = File.read(@file_path, encoding: 'UTF-8')
+          content = remove_underscores(content)
+          normalize_lines(content)
+        end
+
+        private
+
+        def remove_underscores(content)
+          content.gsub(%r{</?u>}, '')
+        end
+
+        def normalize_lines(content)
+          lines = content.split("\n")
+          grouped_subtitles = []
+          current_subtitle = { text: '', start_time: nil, end_time: nil }
+
+          lines.each do |line|
+            if line =~ /^\d+$/ || line.strip.empty?
+              next
+            elsif line =~ /^\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}$/
+              if current_subtitle[:start_time]
+                grouped_subtitles << current_subtitle.clone
+                current_subtitle = { text: '', start_time: nil, end_time: nil }
+              end
+
+              times = line.split(' --> ')
+              current_subtitle[:start_time] = times[0]
+              current_subtitle[:end_time] = times[1]
+            else
+              current_subtitle[:text] += ' ' unless current_subtitle[:text].empty?
+              current_subtitle[:text] += line.strip
+            end
+          end
+
+          grouped_subtitles << current_subtitle unless current_subtitle[:text].empty?
+
+          grouped_subtitles = merge_subtitles(grouped_subtitles)
+
+          build_normalized_content(grouped_subtitles)
+        end
+
+        def merge_subtitles(subtitles)
+          merged_subtitles = []
+          subtitles.each do |subtitle|
+            if merged_subtitles.empty? || merged_subtitles.last[:text] != subtitle[:text]
+              merged_subtitles << subtitle
+            else
+              merged_subtitles.last[:end_time] = subtitle[:end_time]
+            end
+          end
+          merged_subtitles
+        end
+
+        def build_normalized_content(grouped_subtitles)
+          normalized_content = []
+          grouped_subtitles.each_with_index do |subtitle, index|
+            normalized_content << (index + 1).to_s
+            normalized_content << "#{subtitle[:start_time]} --> #{subtitle[:end_time]}"
+            normalized_content << subtitle[:text]
+            normalized_content << ''
+          end
+
+          normalized_content.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/version.rb

@@ -2,6 +2,6 @@
 
 module Appydave
   module Tools
-    VERSION = '0.7.0'
+    VERSION = '0.8.0'
   end
 end

data/lib/appydave/tools.rb

@@ -8,6 +8,7 @@ require 'open3'
 require 'openai'
 require 'optparse'
 require 'k_log'
+require 'pry'
 
 require 'appydave/tools/version'
 require 'appydave/tools/gpt_context/file_collector'
@@ -25,6 +26,8 @@ require 'appydave/tools/bank_reconciliation/clean/read_transactions'
 require 'appydave/tools/bank_reconciliation/clean/mapper'
 require 'appydave/tools/bank_reconciliation/models/transaction'
 
+require 'appydave/tools/subtitle_master/clean'
+
 Appydave::Tools::Configuration::Config.set_default do |config|
   config.config_path = File.expand_path('~/.config/appydave')
   config.register(:settings, Appydave::Tools::Configuration::Models::SettingsConfig)

data/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "appydave-tools",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "appydave-tools",
-      "version": "0.7.0",
+      "version": "0.8.0",
       "devDependencies": {
         "@klueless-js/semantic-release-rubygem": "github:klueless-js/semantic-release-rubygem",
         "@semantic-release/changelog": "^6.0.3",

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appydave-tools",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "AppyDave YouTube Automation Tools",
   "scripts": {
     "release": "semantic-release"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: appydave-tools
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - David Cruwys
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-05-29 00:00:00.000000000 Z
+date: 2024-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: clipboard
@@ -105,6 +105,7 @@ files:
 - bin/console
 - bin/gpt_context.rb
 - bin/setup
+- bin/subtitle_master.rb
 - images.log
 - lib/appydave/tools.rb
 - lib/appydave/tools/bank_reconciliation/_doc.md
@@ -125,6 +126,8 @@ files:
 - lib/appydave/tools/gpt_context/file_collector.rb
 - lib/appydave/tools/name_manager/_doc.md
 - lib/appydave/tools/name_manager/project_name.rb
+- lib/appydave/tools/subtitle_master/_doc.md
+- lib/appydave/tools/subtitle_master/clean.rb
 - lib/appydave/tools/version.rb
 - package-lock.json
 - package.json