appydave-tools 0.10.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/CHANGELOG.md +20 -0
- data/a.json +60 -0
- data/bin/bank_reconciliation.rb +23 -0
- data/bin/gpt_context.rb +37 -30
- data/lib/appydave/tools/gpt_context/file_collector.rb +43 -15
- data/lib/appydave/tools/gpt_context/options.rb +28 -0
- data/lib/appydave/tools/gpt_context/output_handler.rb +36 -0
- data/lib/appydave/tools/{prompt_tools → llm}/models/llm_info.rb +1 -1
- data/lib/appydave/tools/llm/openai_completion.rb +11 -0
- data/lib/appydave/tools/prompt_tools/prompt_completion.rb +2 -2
- data/lib/appydave/tools/version.rb +1 -1
- data/lib/appydave/tools.rb +11 -6
- data/package-lock.json +2 -2
- data/package.json +1 -1
- metadata +7 -9
- data/lib/appydave/tools/bank_reconciliation/_doc.md +0 -36
- data/lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb +0 -145
- data/lib/appydave/tools/bank_reconciliation/clean/mapper.rb +0 -136
- data/lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb +0 -90
- data/lib/appydave/tools/bank_reconciliation/models/transaction.rb +0 -101
- data/lib/appydave/tools/configuration/models/bank_reconciliation_config.rb +0 -97
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 88fcb99717ec45c2efcd0884772e1325a7de9cdca64c58c7e54e8c91203b333d
|
4
|
+
data.tar.gz: d29c8848556ef6c8a189fb3db0b6ad546a0e112d3dc5abffadf3f8e3bbaf058b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a1d033506927b381145b9cd57400e4b6ced89bfee1aeee9bbcf7977b9015bfebfc3116b1d86e1e8de8a185b46180754dabb7bf24b1d4c589e6887fef1db054a8
|
7
|
+
data.tar.gz: abedf296c032a813c0e1abb2bad7eb00b4d4f32c2ee19eda9623648a8a489f6085307f4dbb9763f526d03d86714314d1f613b6c62f848a80ed2c409b64cef5f1
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,23 @@
|
|
1
|
+
## [0.10.4](https://github.com/klueless-io/appydave-tools/compare/v0.10.3...v0.10.4) (2024-10-09)
|
2
|
+
|
3
|
+
|
4
|
+
### Bug Fixes
|
5
|
+
|
6
|
+
* cleanup code base before building documenation ([e473c44](https://github.com/klueless-io/appydave-tools/commit/e473c44546d8f0f5a35461fae263c348e7e2c58f))
|
7
|
+
* cleanup code base before building documenation ([38d8fb4](https://github.com/klueless-io/appydave-tools/commit/38d8fb46929e7c055308aa0b918c18a56fcd5842))
|
8
|
+
* cleanup code base before building documenation ([8dad12a](https://github.com/klueless-io/appydave-tools/commit/8dad12aadb9952f0174df45caed91c9e96070294))
|
9
|
+
* cleanup code base before building documenation ([eddfcc7](https://github.com/klueless-io/appydave-tools/commit/eddfcc78e39c93bfd4de6482690cc66cf90cc54a))
|
10
|
+
* extend prompt tools ([a628d08](https://github.com/klueless-io/appydave-tools/commit/a628d08fcdda521b7d148f0bab3261b879fca07d))
|
11
|
+
* extending bank reconciliation ([b435a20](https://github.com/klueless-io/appydave-tools/commit/b435a20237ef40a8a7d0a4f365ccd3eafdfcce1a))
|
12
|
+
* extending bank reconciliation ([b7878df](https://github.com/klueless-io/appydave-tools/commit/b7878dff0bca4ff1a3af7db19c280a8c245b6c3b))
|
13
|
+
|
14
|
+
## [0.10.3](https://github.com/klueless-io/appydave-tools/compare/v0.10.2...v0.10.3) (2024-06-17)
|
15
|
+
|
16
|
+
|
17
|
+
### Bug Fixes
|
18
|
+
|
19
|
+
* extending bank reconciliation with platform and banking mapping ([848b044](https://github.com/klueless-io/appydave-tools/commit/848b044bf4bb7c27bae6cf33aba400ab68eb105c))
|
20
|
+
|
1
21
|
## [0.10.2](https://github.com/klueless-io/appydave-tools/compare/v0.10.1...v0.10.2) (2024-06-17)
|
2
22
|
|
3
23
|
|
data/a.json
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
{
|
2
|
+
"tree": {
|
3
|
+
"bin": {
|
4
|
+
"gpt_context.rb": {
|
5
|
+
}
|
6
|
+
},
|
7
|
+
"lib": {
|
8
|
+
"appydave": {
|
9
|
+
"tools": {
|
10
|
+
"gpt_context": {
|
11
|
+
"file_collector.rb": {
|
12
|
+
},
|
13
|
+
"options.rb": {
|
14
|
+
},
|
15
|
+
"output_handler.rb": {
|
16
|
+
}
|
17
|
+
}
|
18
|
+
}
|
19
|
+
}
|
20
|
+
},
|
21
|
+
"spec": {
|
22
|
+
"appydave": {
|
23
|
+
"tools": {
|
24
|
+
"gpt_context": {
|
25
|
+
"file_collector_spec.rb": {
|
26
|
+
},
|
27
|
+
"output_handler_spec.rb": {
|
28
|
+
}
|
29
|
+
}
|
30
|
+
}
|
31
|
+
}
|
32
|
+
}
|
33
|
+
},
|
34
|
+
"content": [
|
35
|
+
{
|
36
|
+
"file": "bin/gpt_context.rb",
|
37
|
+
"content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# GPT Chats:\n# https://chatgpt.com/c/670df475-04f4-8002-a758-f5711bf433eb\n\n# Usage:\n# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*.rb'\n# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*' -e 'node_modules/**/*' -e 'package-lock.json' -e 'lib/openai_101/tools/prompts/*.txt'\n#\n# Get GPT Context Gatherer code\n# ./bin/gpt_context.rb -i 'bin/**/*gather*.rb' -i 'lib/openai_101/tools/**/*gather*.rb'\n$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))\n\nrequire 'appydave/tools'\nrequire 'appydave/tools/gpt_context/options'\nrequire 'json'\nrequire 'optparse'\nrequire 'clipboard'\nrequire 'pathname'\n\noptions = Appydave::Tools::GptContext::Options.new(\n working_directory: nil\n)\n\nOptionParser.new do |opts|\n opts.banner = 'Usage: gather_content.rb [options]'\n\n opts.on('-i', '--include PATTERN', 'Pattern or file to include (can be used multiple times)') do |pattern|\n options.include_patterns << pattern\n end\n\n opts.on('-e', '--exclude PATTERN', 'Pattern or file to exclude (can be used multiple times)') do |pattern|\n options.exclude_patterns << pattern\n end\n\n opts.on('-f', '--format FORMAT', 'Output format: content, tree, or json, if not provided then both are used') do |format|\n options.format = format\n end\n\n opts.on('-l', '--line-limit LIMIT', 'Limit the number of lines included from each file') do |limit|\n options.line_limit = limit.to_i\n end\n\n # New option for specifying base directory\n opts.on('-b', '--base-dir DIRECTORY', 'Set the base directory to gather files from') do |directory|\n options.working_directory = directory\n end\n\n # Debug output options\n opts.on('-d', '--debug [MODE]', 'Enable debug mode [none, info, params, debug]', 'none', 'info', 'params', 'debug') do |debug|\n options.debug = debug || 'info'\n end\n\n # Output targets: clipboard or file\n opts.on('-o', '--output TARGET', 'Output target: clipboard, or a file path (can be used multiple times)') do |target|\n options.output_target << target\n end\n\n opts.on_tail('-h', '--help', 'Show this message') do\n puts opts\n puts \"\\nExamples:\"\n puts \" #{File.basename($PROGRAM_NAME)} -i 'lib/**/*.rb' -e 'lib/excluded/**/*.rb' -d\"\n puts \" #{File.basename($PROGRAM_NAME)} --include 'src/**/*.js' --exclude 'src/vendor/**/*.js'\"\n\n puts ''\n puts ' # Get GPT Context Gatherer code that is found in any folder (bin, lib & spec)'\n puts \" #{File.basename($PROGRAM_NAME)} -i '**/*gather*.rb'\"\n exit\n end\nend.parse!\n\nif options.include_patterns.empty? && options.exclude_patterns.empty? && options.format.nil?\n script_name = File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME))\n\n puts 'No options provided to GPT Context. Please specify patterns to include or exclude.'\n puts \"For help, run: #{script_name} --help\"\n exit\nend\n\nif options.output_target.empty?\n puts 'No output target provided. Will default to `clipboard`. You can set the output target using -o'\n options.output_target << 'clipboard'\nend\n\npp options if options.debug == 'params'\n\noptions.working_directory ||= Dir.pwd\n\ngatherer = Appydave::Tools::GptContext::FileCollector.new(options)\ncontent = gatherer.build\n\nif %w[info debug].include?(options.debug)\n puts '-' * 80\n puts content\n puts '-' * 80\nend\n\noutput_handler = Appydave::Tools::GptContext::OutputHandler.new(content, options)\noutput_handler.execute\n\npp options if options.debug == 'debug'\n"
|
38
|
+
},
|
39
|
+
{
|
40
|
+
"file": "lib/appydave/tools/gpt_context/file_collector.rb",
|
41
|
+
"content": "# frozen_string_literal: true\n\nmodule Appydave\n module Tools\n # Build GPT context from various sources\n module GptContext\n # Gathers file names and content based on include and exclude patterns\n class FileCollector\n def initialize(options)\n @options = options\n @include_patterns = options.include_patterns\n @exclude_patterns = options.exclude_patterns\n @format = options.format\n @working_directory = options.working_directory\n @line_limit = options.line_limit\n end\n\n def build\n FileUtils.cd(@working_directory) if @working_directory && Dir.exist?(@working_directory)\n\n formats = @format.split(',')\n result = formats.map do |fmt|\n case fmt\n when 'tree'\n build_tree\n when 'content'\n build_content\n when 'json'\n build_json\n else\n ''\n end\n end.join(\"\\n\\n\")\n\n FileUtils.cd(Dir.home) if @working_directory\n\n result\n end\n\n private\n\n def build_content\n concatenated_content = []\n\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path) || File.directory?(file_path)\n\n content = \"# file: #{file_path}\\n\\n#{read_file_content(file_path)}\"\n concatenated_content << content\n end\n end\n\n concatenated_content.join(\"\\n\\n\")\n end\n\n def read_file_content(file_path)\n lines = File.readlines(file_path)\n return lines.first(@line_limit).join if @line_limit\n\n lines.join\n end\n\n def build_tree\n tree_view = {}\n\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path)\n\n path_parts = file_path.split('/')\n insert_into_tree(tree_view, path_parts)\n end\n end\n\n build_tree_pretty(tree_view).rstrip\n end\n\n def insert_into_tree(tree, path_parts)\n node = tree\n path_parts.each do |part|\n node[part] ||= {}\n node = node[part]\n end\n end\n\n def build_tree_pretty(node, prefix: '', is_last: true, output: ''.dup)\n node.each_with_index do |(part, child), index|\n connector = is_last && index == node.size - 1 ? '└' : '├'\n output << \"#{prefix}#{connector}─ #{part}\\n\"\n next_prefix = is_last && index == node.size - 1 ? ' ' : '│ '\n build_tree_pretty(child, prefix: \"#{prefix}#{next_prefix}\", is_last: child.empty? || index == node.size - 1, output: output)\n end\n output\n end\n\n def build_json\n json_output = {\n 'tree' => {},\n 'content' => []\n }\n\n # Building tree structure in JSON\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path)\n\n path_parts = file_path.split('/')\n insert_into_tree(json_output['tree'], path_parts)\n end\n end\n\n # Building content structure in JSON\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path) || File.directory?(file_path)\n\n json_output['content'] << {\n 'file' => file_path,\n 'content' => read_file_content(file_path)\n }\n end\n end\n\n JSON.pretty_generate(json_output)\n end\n\n def excluded?(file_path)\n @exclude_patterns.any? { |pattern| File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) }\n end\n end\n end\n end\nend\n"
|
42
|
+
},
|
43
|
+
{
|
44
|
+
"file": "lib/appydave/tools/gpt_context/options.rb",
|
45
|
+
"content": "# frozen_string_literal: true\n\nmodule Appydave\n module Tools\n module GptContext\n # Struct with keyword_init: true to allow named parameters\n Options = Struct.new(\n :include_patterns,\n :exclude_patterns,\n :format,\n :line_limit,\n :debug,\n :output_target,\n :working_directory,\n keyword_init: true\n ) do\n def initialize(**args)\n super\n self.include_patterns ||= []\n self.exclude_patterns ||= []\n self.format ||= 'tree,content'\n self.debug ||= 'none'\n self.output_target ||= []\n end\n end\n end\n end\nend\n"
|
46
|
+
},
|
47
|
+
{
|
48
|
+
"file": "lib/appydave/tools/gpt_context/output_handler.rb",
|
49
|
+
"content": "# frozen_string_literal: true\n\nmodule Appydave\n module Tools\n module GptContext\n class OutputHandler\n def initialize(content, options)\n @content = content\n @output_targets = options.output_target\n @working_directory = options.working_directory\n end\n\n def execute\n @output_targets.each do |target|\n case target\n when 'clipboard'\n Clipboard.copy(@content)\n when /^.+$/\n write_to_file(target)\n end\n end\n end\n\n private\n\n attr_reader :content, :output_targets, :working_directory\n\n def write_to_file(target)\n resolved_path = Pathname.new(target).absolute? ? target : File.join(working_directory, target)\n File.write(resolved_path, content)\n end\n end\n end\n end\nend\n"
|
50
|
+
},
|
51
|
+
{
|
52
|
+
"file": "spec/appydave/tools/gpt_context/file_collector_spec.rb",
|
53
|
+
"content": "# frozen_string_literal: true\n\nRSpec.describe Appydave::Tools::GptContext::FileCollector do\n describe '#build' do\n subject { described_class.new(include_patterns: include_patterns, exclude_patterns: exclude_patterns, format: format, line_limit: line_limit) }\n\n let(:include_patterns) { ['spec/fixtures/gpt-content-gatherer/**/*.txt'] }\n let(:exclude_patterns) { ['spec/fixtures/gpt-content-gatherer/excluded/*.txt', '**/deep/**/*'] }\n let(:format) { 'content' }\n let(:line_limit) { nil }\n\n context 'when gathering content' do\n it 'concatenates content from files matching include patterns' do\n expect(subject.build).to include('File 1 content', 'File 2 content')\n end\n\n it 'excludes content from files matching exclude patterns' do\n expect(subject.build).not_to include('Excluded file content')\n expect(subject.build).not_to include('Deep 1')\n expect(subject.build).not_to include('Deep 1')\n end\n\n it 'includes file paths as headers in the gathered content' do\n expect(subject.build)\n .to include('# file: spec/fixtures/gpt-content-gatherer/included/file1.txt')\n .and include('# file: spec/fixtures/gpt-content-gatherer/included/file2.txt')\n end\n end\n\n context 'when line limit is set' do\n let(:line_limit) { 1 }\n\n it 'limits the number of lines included from each file' do\n expect(subject.build).not_to include('Line #2')\n end\n end\n end\n\n describe '#build with tree format' do\n subject { described_class.new(include_patterns: include_patterns, exclude_patterns: exclude_patterns, format: 'tree') }\n\n let(:include_patterns) { ['spec/fixtures/gpt-content-gatherer/**/*'] }\n let(:exclude_patterns) { [] }\n\n before do\n allow(Dir).to receive(:glob).and_return(\n [\n 'spec/fixtures/gpt-content-gatherer/included/file1.txt',\n 'spec/fixtures/gpt-content-gatherer/included/file2.txt',\n 'spec/fixtures/gpt-content-gatherer/included/subdir/file3.txt'\n ]\n )\n allow(File).to receive(:directory?).and_return(false)\n end\n\n it 'prints a tree view of the included files and directories with improved ASCII art' do\n expected_output = <<~TREE\n └─ spec\n └─ fixtures\n └─ gpt-content-gatherer\n └─ included\n ├─ file1.txt\n ├─ file2.txt\n └─ subdir\n └─ file3.txt\n TREE\n\n expect(subject.build.strip).to eq(expected_output.strip)\n end\n end\n\n describe '#build with both formats' do\n subject { described_class.new(include_patterns: include_patterns, exclude_patterns: exclude_patterns, format: 'tree,content') }\n\n let(:include_patterns) { ['spec/fixtures/gpt-content-gatherer/**/*'] }\n let(:exclude_patterns) { [] }\n\n before do\n allow(Dir).to receive(:glob).and_return(\n [\n 'spec/fixtures/gpt-content-gatherer/included/file1.txt',\n 'spec/fixtures/gpt-content-gatherer/included/file2.txt',\n 'spec/fixtures/gpt-content-gatherer/included/subdir/file3.txt'\n ]\n )\n allow(File).to receive(:directory?).and_return(false)\n end\n\n it 'prints both a tree view and the file contents' do\n expected_output_tree = <<~TREE\n └─ spec\n └─ fixtures\n └─ gpt-content-gatherer\n └─ included\n ├─ file1.txt\n ├─ file2.txt\n └─ subdir\n └─ file3.txt\n TREE\n\n expected_output_content = <<~CONTENT\n # file: spec/fixtures/gpt-content-gatherer/included/file1.txt\n\n File 1 content\n Line #2\n\n # file: spec/fixtures/gpt-content-gatherer/included/file2.txt\n\n File 2 content\n Line #2\n\n # file: spec/fixtures/gpt-content-gatherer/included/subdir/file3.txt\n\n File 3 content\n Line #2\n CONTENT\n\n expect(subject.build).to include(expected_output_tree.strip)\n expect(subject.build).to include(expected_output_content.strip)\n end\n end\nend\n"
|
54
|
+
},
|
55
|
+
{
|
56
|
+
"file": "spec/appydave/tools/gpt_context/output_handler_spec.rb",
|
57
|
+
"content": "# frozen_string_literal: true\n\nRSpec.describe Appydave::Tools::GptContext::OutputHandler do\n subject { described_class.new(content, options) }\n\n let(:content) { 'Sample content' }\n let(:options) { double('Options', output_target: ['clipboard'], working_directory: Dir.pwd) }\n\n describe '#execute' do\n it 'copies content to clipboard when output target is clipboard' do\n expect(Clipboard).to receive(:copy).with(content)\n subject.execute\n end\n\n context 'when output target is a file' do\n let(:options) { double('Options', output_target: ['output.txt'], working_directory: Dir.pwd) }\n\n it 'writes content to the specified file' do\n expect(File).to receive(:write).with(File.join(Dir.pwd, 'output.txt'), content)\n subject.execute\n end\n end\n\n context 'when multiple output targets are specified' do\n let(:options) { double('Options', output_target: ['clipboard', 'output.txt'], working_directory: Dir.pwd) }\n\n it 'copies content to clipboard and writes to file' do\n expect(Clipboard).to receive(:copy).with(content)\n expect(File).to receive(:write).with(File.join(Dir.pwd, 'output.txt'), content)\n subject.execute\n end\n end\n end\nend\n"
|
58
|
+
}
|
59
|
+
]
|
60
|
+
}
|
data/bin/bank_reconciliation.rb
CHANGED
@@ -14,6 +14,7 @@ class BankReconciliationCLI
|
|
14
14
|
def initialize
|
15
15
|
@commands = {
|
16
16
|
'clean' => method(:clean_transactions),
|
17
|
+
'transform' => method(:transform),
|
17
18
|
'process' => method(:process_transactions),
|
18
19
|
'filter' => method(:filter_transactions)
|
19
20
|
}
|
@@ -73,6 +74,28 @@ class BankReconciliationCLI
|
|
73
74
|
cleaner.clean_transactions(include_patterns, output_file)
|
74
75
|
end
|
75
76
|
|
77
|
+
def transform(args)
|
78
|
+
options = {}
|
79
|
+
OptionParser.new do |opts|
|
80
|
+
opts.banner = 'Usage: bank_reconciliation.rb clean [options]'
|
81
|
+
|
82
|
+
opts.on('-c', '--to-csv', 'Write chart of accounts JSON to CSV') { options[:to_csv] = true }
|
83
|
+
opts.on('-j', '--to-json', 'Write chart of accounts CSV to JSON') { options[:to_json] = true }
|
84
|
+
|
85
|
+
opts.on('-d', '--debug', 'Enable debug mode') do
|
86
|
+
options[:debug] = true
|
87
|
+
end
|
88
|
+
|
89
|
+
opts.on_tail('-h', '--help', 'Show this message') do
|
90
|
+
puts opts
|
91
|
+
exit
|
92
|
+
end
|
93
|
+
end.parse!(args)
|
94
|
+
|
95
|
+
Appydave::Tools::Configuration::Models::BankReconciliationConfig.new.coa_to_csv if options[:to_csv]
|
96
|
+
Appydave::Tools::Configuration::Models::BankReconciliationConfig.new.coa_csv_to_json if options[:to_json]
|
97
|
+
end
|
98
|
+
|
76
99
|
def process_transactions(args)
|
77
100
|
options = {}
|
78
101
|
OptionParser.new do |opts|
|
data/bin/gpt_context.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
|
+
# GPT Chats:
|
5
|
+
# https://chatgpt.com/c/670df475-04f4-8002-a758-f5711bf433eb
|
6
|
+
|
4
7
|
# Usage:
|
5
8
|
# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*.rb'
|
6
9
|
# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*' -e 'node_modules/**/*' -e 'package-lock.json' -e 'lib/openai_101/tools/prompts/*.txt'
|
@@ -11,39 +14,42 @@ $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
|
11
14
|
|
12
15
|
require 'appydave/tools'
|
13
16
|
|
14
|
-
options =
|
15
|
-
|
16
|
-
|
17
|
-
format: 'tree,content',
|
18
|
-
line_limit: nil,
|
19
|
-
debug: 'none'
|
20
|
-
}
|
17
|
+
options = Appydave::Tools::GptContext::Options.new(
|
18
|
+
working_directory: nil
|
19
|
+
)
|
21
20
|
|
22
21
|
OptionParser.new do |opts|
|
23
22
|
opts.banner = 'Usage: gather_content.rb [options]'
|
24
23
|
|
25
24
|
opts.on('-i', '--include PATTERN', 'Pattern or file to include (can be used multiple times)') do |pattern|
|
26
|
-
options
|
25
|
+
options.include_patterns << pattern
|
27
26
|
end
|
28
27
|
|
29
28
|
opts.on('-e', '--exclude PATTERN', 'Pattern or file to exclude (can be used multiple times)') do |pattern|
|
30
|
-
options
|
29
|
+
options.exclude_patterns << pattern
|
31
30
|
end
|
32
31
|
|
33
|
-
opts.on('-f', '--format FORMAT', 'Output format: content or
|
34
|
-
options
|
32
|
+
opts.on('-f', '--format FORMAT', 'Output format: content, tree, or json, if not provided then both are used') do |format|
|
33
|
+
options.format = format
|
35
34
|
end
|
36
35
|
|
37
36
|
opts.on('-l', '--line-limit LIMIT', 'Limit the number of lines included from each file') do |limit|
|
38
|
-
options
|
37
|
+
options.line_limit = limit.to_i
|
39
38
|
end
|
40
39
|
|
41
|
-
#
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
40
|
+
# New option for specifying base directory
|
41
|
+
opts.on('-b', '--base-dir DIRECTORY', 'Set the base directory to gather files from') do |directory|
|
42
|
+
options.working_directory = directory
|
43
|
+
end
|
44
|
+
|
45
|
+
# Debug output options
|
46
|
+
opts.on('-d', '--debug [MODE]', 'Enable debug mode [none, info, params, debug]', 'none', 'info', 'params', 'debug') do |debug|
|
47
|
+
options.debug = debug || 'info'
|
48
|
+
end
|
49
|
+
|
50
|
+
# Output targets: clipboard or file
|
51
|
+
opts.on('-o', '--output TARGET', 'Output target: clipboard, or a file path (can be used multiple times)') do |target|
|
52
|
+
options.output_target << target
|
47
53
|
end
|
48
54
|
|
49
55
|
opts.on_tail('-h', '--help', 'Show this message') do
|
@@ -59,7 +65,7 @@ OptionParser.new do |opts|
|
|
59
65
|
end
|
60
66
|
end.parse!
|
61
67
|
|
62
|
-
if options
|
68
|
+
if options.include_patterns.empty? && options.exclude_patterns.empty? && options.format.nil?
|
63
69
|
script_name = File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME))
|
64
70
|
|
65
71
|
puts 'No options provided to GPT Context. Please specify patterns to include or exclude.'
|
@@ -67,24 +73,25 @@ if options[:include_patterns].empty? && options[:exclude_patterns].empty? && opt
|
|
67
73
|
exit
|
68
74
|
end
|
69
75
|
|
70
|
-
|
76
|
+
if options.output_target.empty?
|
77
|
+
puts 'No output target provided. Will default to `clipboard`. You can set the output target using -o'
|
78
|
+
options.output_target << 'clipboard'
|
79
|
+
end
|
80
|
+
|
81
|
+
pp options if options.debug == 'params'
|
71
82
|
|
72
|
-
|
73
|
-
include_patterns: options[:include_patterns],
|
74
|
-
exclude_patterns: options[:exclude_patterns],
|
75
|
-
format: options[:format],
|
76
|
-
line_limit: options[:line_limit],
|
77
|
-
working_directory: Dir.pwd
|
78
|
-
)
|
83
|
+
options.working_directory ||= Dir.pwd
|
79
84
|
|
85
|
+
gatherer = Appydave::Tools::GptContext::FileCollector.new(options)
|
80
86
|
content = gatherer.build
|
81
87
|
|
82
|
-
if %w[
|
88
|
+
if %w[info debug].include?(options.debug)
|
83
89
|
puts '-' * 80
|
84
90
|
puts content
|
85
91
|
puts '-' * 80
|
86
92
|
end
|
87
93
|
|
88
|
-
|
94
|
+
output_handler = Appydave::Tools::GptContext::OutputHandler.new(content, options)
|
95
|
+
output_handler.execute
|
89
96
|
|
90
|
-
|
97
|
+
pp options if options.debug == 'debug'
|
@@ -6,32 +6,33 @@ module Appydave
|
|
6
6
|
module GptContext
|
7
7
|
# Gathers file names and content based on include and exclude patterns
|
8
8
|
class FileCollector
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
@
|
13
|
-
@
|
14
|
-
@
|
15
|
-
@
|
16
|
-
@line_limit = line_limit
|
9
|
+
def initialize(options)
|
10
|
+
@options = options
|
11
|
+
@include_patterns = options.include_patterns
|
12
|
+
@exclude_patterns = options.exclude_patterns
|
13
|
+
@format = options.format
|
14
|
+
@working_directory = options.working_directory
|
15
|
+
@line_limit = options.line_limit
|
17
16
|
end
|
18
17
|
|
19
18
|
def build
|
20
|
-
FileUtils.cd(working_directory) if working_directory && Dir.exist?(working_directory)
|
19
|
+
FileUtils.cd(@working_directory) if @working_directory && Dir.exist?(@working_directory)
|
21
20
|
|
22
|
-
formats = format.split(',')
|
21
|
+
formats = @format.split(',')
|
23
22
|
result = formats.map do |fmt|
|
24
23
|
case fmt
|
25
24
|
when 'tree'
|
26
25
|
build_tree
|
27
26
|
when 'content'
|
28
27
|
build_content
|
28
|
+
when 'json'
|
29
|
+
build_json
|
29
30
|
else
|
30
31
|
''
|
31
32
|
end
|
32
33
|
end.join("\n\n")
|
33
34
|
|
34
|
-
FileUtils.cd(Dir.home) if working_directory
|
35
|
+
FileUtils.cd(Dir.home) if @working_directory
|
35
36
|
|
36
37
|
result
|
37
38
|
end
|
@@ -41,7 +42,7 @@ module Appydave
|
|
41
42
|
def build_content
|
42
43
|
concatenated_content = []
|
43
44
|
|
44
|
-
include_patterns.each do |pattern|
|
45
|
+
@include_patterns.each do |pattern|
|
45
46
|
Dir.glob(pattern).each do |file_path|
|
46
47
|
next if excluded?(file_path) || File.directory?(file_path)
|
47
48
|
|
@@ -55,7 +56,7 @@ module Appydave
|
|
55
56
|
|
56
57
|
def read_file_content(file_path)
|
57
58
|
lines = File.readlines(file_path)
|
58
|
-
return lines.first(line_limit).join if line_limit
|
59
|
+
return lines.first(@line_limit).join if @line_limit
|
59
60
|
|
60
61
|
lines.join
|
61
62
|
end
|
@@ -63,7 +64,7 @@ module Appydave
|
|
63
64
|
def build_tree
|
64
65
|
tree_view = {}
|
65
66
|
|
66
|
-
include_patterns.each do |pattern|
|
67
|
+
@include_patterns.each do |pattern|
|
67
68
|
Dir.glob(pattern).each do |file_path|
|
68
69
|
next if excluded?(file_path)
|
69
70
|
|
@@ -93,8 +94,35 @@ module Appydave
|
|
93
94
|
output
|
94
95
|
end
|
95
96
|
|
97
|
+
def build_json
|
98
|
+
json_output = {
|
99
|
+
'tree' => {},
|
100
|
+
'content' => []
|
101
|
+
}
|
102
|
+
|
103
|
+
# Building tree structure in JSON
|
104
|
+
@include_patterns.each do |pattern|
|
105
|
+
Dir.glob(pattern).each do |file_path|
|
106
|
+
next if excluded?(file_path)
|
107
|
+
|
108
|
+
path_parts = file_path.split('/')
|
109
|
+
insert_into_tree(json_output['tree'], path_parts)
|
110
|
+
|
111
|
+
# Building content structure in JSON
|
112
|
+
next if excluded?(file_path) || File.directory?(file_path)
|
113
|
+
|
114
|
+
json_output['content'] << {
|
115
|
+
'file' => file_path,
|
116
|
+
'content' => read_file_content(file_path)
|
117
|
+
}
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
JSON.pretty_generate(json_output)
|
122
|
+
end
|
123
|
+
|
96
124
|
def excluded?(file_path)
|
97
|
-
exclude_patterns.any? { |pattern| File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) }
|
125
|
+
@exclude_patterns.any? { |pattern| File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) }
|
98
126
|
end
|
99
127
|
end
|
100
128
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Appydave
|
4
|
+
module Tools
|
5
|
+
module GptContext
|
6
|
+
# Struct with keyword_init: true to allow named parameters
|
7
|
+
Options = Struct.new(
|
8
|
+
:include_patterns,
|
9
|
+
:exclude_patterns,
|
10
|
+
:format,
|
11
|
+
:line_limit,
|
12
|
+
:debug,
|
13
|
+
:output_target,
|
14
|
+
:working_directory,
|
15
|
+
keyword_init: true
|
16
|
+
) do
|
17
|
+
def initialize(**args)
|
18
|
+
super
|
19
|
+
self.include_patterns ||= []
|
20
|
+
self.exclude_patterns ||= []
|
21
|
+
self.format ||= 'tree,content'
|
22
|
+
self.debug ||= 'none'
|
23
|
+
self.output_target ||= []
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Appydave
|
4
|
+
module Tools
|
5
|
+
module GptContext
|
6
|
+
# OutputHandler is responsible for writing the output to the desired target
|
7
|
+
class OutputHandler
|
8
|
+
def initialize(content, options)
|
9
|
+
@content = content
|
10
|
+
@output_targets = options.output_target
|
11
|
+
@working_directory = options.working_directory
|
12
|
+
end
|
13
|
+
|
14
|
+
def execute
|
15
|
+
@output_targets.each do |target|
|
16
|
+
case target
|
17
|
+
when 'clipboard'
|
18
|
+
Clipboard.copy(@content)
|
19
|
+
when /^.+$/
|
20
|
+
write_to_file(target)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
attr_reader :content, :output_targets, :working_directory
|
28
|
+
|
29
|
+
def write_to_file(target)
|
30
|
+
resolved_path = Pathname.new(target).absolute? ? target : File.join(working_directory, target)
|
31
|
+
File.write(resolved_path, content)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -16,7 +16,7 @@ module Appydave
|
|
16
16
|
attr_reader :clipboard
|
17
17
|
|
18
18
|
def initialize(options = {})
|
19
|
-
|
19
|
+
setup_options(options)
|
20
20
|
|
21
21
|
validate_options
|
22
22
|
end
|
@@ -44,7 +44,7 @@ module Appydave
|
|
44
44
|
|
45
45
|
private
|
46
46
|
|
47
|
-
def
|
47
|
+
def setup_options(options)
|
48
48
|
@prompt = options.delete(:prompt)
|
49
49
|
@prompt_file = options.delete(:prompt_file)
|
50
50
|
@llm = Appydave::Tools::PromptTools::Models::LlmInfo.new(
|
data/lib/appydave/tools.rb
CHANGED
@@ -7,6 +7,7 @@ require 'json'
|
|
7
7
|
require 'open3'
|
8
8
|
require 'openai'
|
9
9
|
require 'optparse'
|
10
|
+
require 'pathname'
|
10
11
|
require 'k_log'
|
11
12
|
require 'active_model'
|
12
13
|
|
@@ -24,6 +25,9 @@ require 'appydave/tools/types/hash_type'
|
|
24
25
|
require 'appydave/tools/types/array_type'
|
25
26
|
require 'appydave/tools/types/base_model'
|
26
27
|
|
28
|
+
require 'appydave/tools/llm/models/llm_info'
|
29
|
+
require 'appydave/tools/llm/openai_completion'
|
30
|
+
|
27
31
|
require 'appydave/tools/cli_actions/base_action'
|
28
32
|
|
29
33
|
# May want to move this into the tools location
|
@@ -31,23 +35,24 @@ require 'appydave/tools/cli_actions/prompt_completion_action'
|
|
31
35
|
require 'appydave/tools/cli_actions/get_video_action'
|
32
36
|
require 'appydave/tools/cli_actions/update_video_action'
|
33
37
|
|
38
|
+
require 'appydave/tools/gpt_context/options'
|
34
39
|
require 'appydave/tools/gpt_context/file_collector'
|
40
|
+
require 'appydave/tools/gpt_context/output_handler'
|
35
41
|
|
36
42
|
require 'appydave/tools/configuration/openai'
|
37
43
|
require 'appydave/tools/configuration/configurable'
|
38
44
|
require 'appydave/tools/configuration/config'
|
39
45
|
require 'appydave/tools/configuration/models/config_base'
|
40
46
|
require 'appydave/tools/configuration/models/settings_config'
|
41
|
-
require 'appydave/tools/configuration/models/bank_reconciliation_config'
|
47
|
+
# require 'appydave/tools/configuration/models/bank_reconciliation_config'
|
42
48
|
require 'appydave/tools/configuration/models/channels_config'
|
43
49
|
require 'appydave/tools/configuration/models/youtube_automation_config'
|
44
50
|
require 'appydave/tools/name_manager/project_name'
|
45
|
-
require 'appydave/tools/bank_reconciliation/clean/clean_transactions'
|
46
|
-
require 'appydave/tools/bank_reconciliation/clean/read_transactions'
|
47
|
-
require 'appydave/tools/bank_reconciliation/clean/mapper'
|
48
|
-
require 'appydave/tools/bank_reconciliation/models/transaction'
|
51
|
+
# require 'appydave/tools/bank_reconciliation/clean/clean_transactions'
|
52
|
+
# require 'appydave/tools/bank_reconciliation/clean/read_transactions'
|
53
|
+
# require 'appydave/tools/bank_reconciliation/clean/mapper'
|
54
|
+
# require 'appydave/tools/bank_reconciliation/models/transaction'
|
49
55
|
|
50
|
-
require 'appydave/tools/prompt_tools/models/llm_info'
|
51
56
|
require 'appydave/tools/prompt_tools/prompt_completion'
|
52
57
|
|
53
58
|
require 'appydave/tools/subtitle_master/clean'
|
data/package-lock.json
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
{
|
2
2
|
"name": "appydave-tools",
|
3
|
-
"version": "0.
|
3
|
+
"version": "0.11.0",
|
4
4
|
"lockfileVersion": 3,
|
5
5
|
"requires": true,
|
6
6
|
"packages": {
|
7
7
|
"": {
|
8
8
|
"name": "appydave-tools",
|
9
|
-
"version": "0.
|
9
|
+
"version": "0.11.0",
|
10
10
|
"devDependencies": {
|
11
11
|
"@klueless-js/semantic-release-rubygem": "github:klueless-js/semantic-release-rubygem",
|
12
12
|
"@semantic-release/changelog": "^6.0.3",
|
data/package.json
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: appydave-tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Cruwys
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-10-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|
@@ -156,6 +156,7 @@ files:
|
|
156
156
|
- LICENSE.txt
|
157
157
|
- README.md
|
158
158
|
- Rakefile
|
159
|
+
- a.json
|
159
160
|
- bin/bank_reconciliation.rb
|
160
161
|
- bin/configuration.rb
|
161
162
|
- bin/console
|
@@ -167,11 +168,6 @@ files:
|
|
167
168
|
- bin/youtube_manager.rb
|
168
169
|
- images.log
|
169
170
|
- lib/appydave/tools.rb
|
170
|
-
- lib/appydave/tools/bank_reconciliation/_doc.md
|
171
|
-
- lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb
|
172
|
-
- lib/appydave/tools/bank_reconciliation/clean/mapper.rb
|
173
|
-
- lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb
|
174
|
-
- lib/appydave/tools/bank_reconciliation/models/transaction.rb
|
175
171
|
- lib/appydave/tools/cli_actions/_doc.md
|
176
172
|
- lib/appydave/tools/cli_actions/base_action.rb
|
177
173
|
- lib/appydave/tools/cli_actions/get_video_action.rb
|
@@ -180,7 +176,6 @@ files:
|
|
180
176
|
- lib/appydave/tools/configuration/_doc.md
|
181
177
|
- lib/appydave/tools/configuration/config.rb
|
182
178
|
- lib/appydave/tools/configuration/configurable.rb
|
183
|
-
- lib/appydave/tools/configuration/models/bank_reconciliation_config.rb
|
184
179
|
- lib/appydave/tools/configuration/models/channels_config.rb
|
185
180
|
- lib/appydave/tools/configuration/models/config_base.rb
|
186
181
|
- lib/appydave/tools/configuration/models/settings_config copy.xrb
|
@@ -190,10 +185,13 @@ files:
|
|
190
185
|
- lib/appydave/tools/debuggable.rb
|
191
186
|
- lib/appydave/tools/gpt_context/_doc.md
|
192
187
|
- lib/appydave/tools/gpt_context/file_collector.rb
|
188
|
+
- lib/appydave/tools/gpt_context/options.rb
|
189
|
+
- lib/appydave/tools/gpt_context/output_handler.rb
|
190
|
+
- lib/appydave/tools/llm/models/llm_info.rb
|
191
|
+
- lib/appydave/tools/llm/openai_completion.rb
|
193
192
|
- lib/appydave/tools/name_manager/_doc.md
|
194
193
|
- lib/appydave/tools/name_manager/project_name.rb
|
195
194
|
- lib/appydave/tools/prompt_tools/_doc.md
|
196
|
-
- lib/appydave/tools/prompt_tools/models/llm_info.rb
|
197
195
|
- lib/appydave/tools/prompt_tools/prompt_completion.rb
|
198
196
|
- lib/appydave/tools/subtitle_master/_doc.md
|
199
197
|
- lib/appydave/tools/subtitle_master/clean.rb
|
@@ -1,36 +0,0 @@
|
|
1
|
-
# Bank reconciliation
|
2
|
-
|
3
|
-
[ChatGPT conversation](https://chatgpt.com/c/5d382562-95e5-4243-9b74-c3807d363486)
|
4
|
-
|
5
|
-
|
6
|
-
## Code structure
|
7
|
-
|
8
|
-
```bash
|
9
|
-
├─ lib
|
10
|
-
│ ├─ appydave
|
11
|
-
│ │ └─ tools
|
12
|
-
│ │ ├─ bank_reconciliation
|
13
|
-
│ │ │ ├─ clean
|
14
|
-
│ │ │ │ ├─ read_transactions.rb
|
15
|
-
│ │ │ │ ├─ transaction_cleaner.rb
|
16
|
-
│ │ │ ├─ models
|
17
|
-
│ │ │ │ ├─ raw_transaction.rb
|
18
|
-
│ │ │ │ └─ reconciled_transaction.rb
|
19
|
-
│ │ └─ configuration
|
20
|
-
│ │ └─ models
|
21
|
-
│ │ └─ bank_reconciliation_config.rb
|
22
|
-
└─ spec
|
23
|
-
├─ appydave
|
24
|
-
│ ├─ tools
|
25
|
-
│ │ ├─ bank_reconciliation
|
26
|
-
│ │ │ ├─ clean
|
27
|
-
│ │ │ │ ├─ read_transactions_spec.rb
|
28
|
-
│ │ │ ├─ models
|
29
|
-
│ │ │ │ └─ raw_transaction_spec.rb
|
30
|
-
│ │ └─ configuration
|
31
|
-
│ │ └─ models
|
32
|
-
│ │ └─ bank_reconciliation_config_spec.rb
|
33
|
-
└─ fixtures
|
34
|
-
└─ bank-reconciliation
|
35
|
-
└─ bank-west.csv
|
36
|
-
```
|
@@ -1,145 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Clean
|
7
|
-
# Clean transactions
|
8
|
-
class CleanTransactions
|
9
|
-
include KLog::Logging
|
10
|
-
include Appydave::Tools::Configuration::Configurable
|
11
|
-
include Appydave::Tools::Debuggable
|
12
|
-
|
13
|
-
attr_reader :transaction_folder
|
14
|
-
attr_reader :output_folder
|
15
|
-
attr_reader :transactions
|
16
|
-
|
17
|
-
# (config_file)
|
18
|
-
def initialize(transaction_folder: nil, output_folder: nil, debug: false)
|
19
|
-
@debug = debug
|
20
|
-
# needs to use config.bank_reconciliation.transaction_folder
|
21
|
-
transaction_folder ||= '/Volumes/Expansion/Sync/bank-reconciliation/original-transactions'
|
22
|
-
output_folder ||= File.join(transaction_folder, 'clean')
|
23
|
-
|
24
|
-
@transaction_folder = transaction_folder
|
25
|
-
@output_folder = output_folder
|
26
|
-
end
|
27
|
-
|
28
|
-
def clean_transactions(input_globs, output_file)
|
29
|
-
log_info("Starting transaction cleaning with input patterns: #{input_globs}")
|
30
|
-
|
31
|
-
raw_transactions = grab_raw_transactions(input_globs)
|
32
|
-
log_info("Total raw transactions collected: #{raw_transactions.size}")
|
33
|
-
|
34
|
-
transactions, duplicates_count = deduplicate_across_files(raw_transactions)
|
35
|
-
log_info("Duplicates found and removed: #{duplicates_count}")
|
36
|
-
|
37
|
-
transactions = Mapper.new.map(transactions)
|
38
|
-
log_info('Transactions mapped to chart of accounts and bank accounts')
|
39
|
-
|
40
|
-
log_kv 'Deduped consolidated transactions', duplicates_count if duplicates_count.positive?
|
41
|
-
|
42
|
-
save_to_csv(transactions, output_file)
|
43
|
-
log_kv('Transaction Output File', full_output_file(output_file))
|
44
|
-
|
45
|
-
@transactions = transactions
|
46
|
-
end
|
47
|
-
|
48
|
-
private
|
49
|
-
|
50
|
-
def grab_raw_transactions(input_globs)
|
51
|
-
original_dir = Dir.pwd
|
52
|
-
transactions = []
|
53
|
-
|
54
|
-
begin
|
55
|
-
Dir.chdir(transaction_folder)
|
56
|
-
|
57
|
-
input_globs.each do |glob|
|
58
|
-
Dir.glob(glob).each do |file|
|
59
|
-
log_kv 'Reading transactions from', file
|
60
|
-
raw_transactions = ReadTransactions.new(file).read
|
61
|
-
deduped_transactions, duplicates_count = deduplicate_within_file(raw_transactions)
|
62
|
-
|
63
|
-
if duplicates_count.positive?
|
64
|
-
log_kv 'Duplicates count within file', duplicates_count
|
65
|
-
log_kv 'File', file
|
66
|
-
end
|
67
|
-
|
68
|
-
transactions += deduped_transactions
|
69
|
-
end
|
70
|
-
end
|
71
|
-
ensure
|
72
|
-
Dir.chdir(original_dir)
|
73
|
-
end
|
74
|
-
|
75
|
-
transactions
|
76
|
-
end
|
77
|
-
|
78
|
-
def deduplicate_within_file(transactions)
|
79
|
-
unique_transactions = transactions.uniq do |transaction|
|
80
|
-
[
|
81
|
-
transaction.bsb_number,
|
82
|
-
transaction.account_number,
|
83
|
-
transaction.transaction_date,
|
84
|
-
transaction.narration,
|
85
|
-
transaction.cheque_number,
|
86
|
-
transaction.debit,
|
87
|
-
transaction.credit,
|
88
|
-
transaction.balance,
|
89
|
-
transaction.transaction_type
|
90
|
-
]
|
91
|
-
end
|
92
|
-
|
93
|
-
duplicates = transactions.size - unique_transactions.size
|
94
|
-
|
95
|
-
[unique_transactions, duplicates]
|
96
|
-
end
|
97
|
-
|
98
|
-
def deduplicate_across_files(transactions)
|
99
|
-
grouped_transactions = transactions.group_by do |transaction|
|
100
|
-
[
|
101
|
-
transaction.bsb_number,
|
102
|
-
transaction.account_number,
|
103
|
-
transaction.transaction_date,
|
104
|
-
transaction.narration,
|
105
|
-
transaction.cheque_number,
|
106
|
-
transaction.debit,
|
107
|
-
transaction.credit,
|
108
|
-
transaction.balance,
|
109
|
-
transaction.transaction_type
|
110
|
-
]
|
111
|
-
end
|
112
|
-
|
113
|
-
unique_transactions = []
|
114
|
-
duplicates_count = 0
|
115
|
-
|
116
|
-
grouped_transactions.each_value do |dupes|
|
117
|
-
unique_transaction = dupes.first
|
118
|
-
unique_transaction.source_files = dupes.flat_map(&:source_files).uniq
|
119
|
-
duplicates_count += dupes.size - 1
|
120
|
-
unique_transactions << unique_transaction
|
121
|
-
end
|
122
|
-
|
123
|
-
[unique_transactions, duplicates_count]
|
124
|
-
end
|
125
|
-
|
126
|
-
def full_output_file(output_file)
|
127
|
-
File.join(output_folder, output_file)
|
128
|
-
end
|
129
|
-
|
130
|
-
def save_to_csv(transactions, output_file)
|
131
|
-
FileUtils.mkdir_p(output_folder)
|
132
|
-
output_file = full_output_file(output_file)
|
133
|
-
|
134
|
-
CSV.open(output_file, 'w') do |csv|
|
135
|
-
csv << Appydave::Tools::BankReconciliation::Models::Transaction.csv_headers
|
136
|
-
transactions.each do |transaction|
|
137
|
-
csv << transaction.to_csv_row
|
138
|
-
end
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
143
|
-
end
|
144
|
-
end
|
145
|
-
end
|
@@ -1,136 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Clean
|
7
|
-
# Map transactions to chart of accounts and bank accounts
|
8
|
-
class Mapper
|
9
|
-
include Appydave::Tools::Configuration::Configurable
|
10
|
-
|
11
|
-
# "bank_accounts": [
|
12
|
-
# {
|
13
|
-
# "account_number": "5435 6859 0116 7736",
|
14
|
-
# "bsb": "",
|
15
|
-
# "name": "Mastercard",
|
16
|
-
# "platform": "Bankwest"
|
17
|
-
# },
|
18
|
-
# {
|
19
|
-
# "account_number": "303-092",
|
20
|
-
# "bsb": "1361644",
|
21
|
-
# "name": "atcall",
|
22
|
-
# "platform": "Bankwest"
|
23
|
-
# },
|
24
|
-
|
25
|
-
def map(transactions)
|
26
|
-
transactions.map do |original_transaction|
|
27
|
-
transaction = original_transaction.dup
|
28
|
-
|
29
|
-
transaction = map_chart_of_account(transaction)
|
30
|
-
map_bank_account(transaction)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
private
|
35
|
-
|
36
|
-
def map_chart_of_account(transaction)
|
37
|
-
equality_match(transaction) ||
|
38
|
-
trigram_match(transaction, 0.9, '90%') ||
|
39
|
-
trigram_match(transaction, 0.8, '80%') ||
|
40
|
-
trigram_match(transaction, 0.7, '70%') ||
|
41
|
-
trigram_match(transaction, 0.6, '60%') ||
|
42
|
-
trigram_match(transaction, 0.5, '50%') ||
|
43
|
-
start_with_match(transaction) ||
|
44
|
-
includes(transaction)
|
45
|
-
transaction
|
46
|
-
end
|
47
|
-
|
48
|
-
def map_bank_account(transaction)
|
49
|
-
bank_account = config.bank_reconciliation.get_bank_account(transaction.account_number, transaction.bsb_number)
|
50
|
-
|
51
|
-
if bank_account
|
52
|
-
transaction.account_name = bank_account.name
|
53
|
-
transaction.platform = bank_account.platform
|
54
|
-
end
|
55
|
-
|
56
|
-
transaction
|
57
|
-
end
|
58
|
-
|
59
|
-
def equality_match(transaction)
|
60
|
-
coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
|
61
|
-
chart_of_account.narration.to_s.delete(' ').downcase == transaction.narration.delete(' ').downcase
|
62
|
-
end
|
63
|
-
|
64
|
-
return nil unless coa
|
65
|
-
|
66
|
-
transaction.coa_match_type = 'equality'
|
67
|
-
transaction.coa_code = coa.code
|
68
|
-
transaction
|
69
|
-
end
|
70
|
-
|
71
|
-
def start_with_match(transaction)
|
72
|
-
coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
|
73
|
-
transaction.narration.to_s.delete(' ').downcase.start_with?(chart_of_account.narration.to_s.downcase)
|
74
|
-
end
|
75
|
-
|
76
|
-
return nil unless coa
|
77
|
-
|
78
|
-
transaction.coa_match_type = 'starts_with'
|
79
|
-
transaction.coa_code = coa.code
|
80
|
-
transaction
|
81
|
-
end
|
82
|
-
|
83
|
-
def includes(transaction)
|
84
|
-
coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
|
85
|
-
transaction.narration.to_s.delete(' ').downcase.include?(chart_of_account.narration.delete(' ').to_s.downcase)
|
86
|
-
end
|
87
|
-
|
88
|
-
return nil unless coa
|
89
|
-
|
90
|
-
transaction.coa_match_type = 'includes'
|
91
|
-
transaction.coa_code = coa.code
|
92
|
-
transaction
|
93
|
-
end
|
94
|
-
|
95
|
-
def trigram_match(transaction, score_threshold, match_type)
|
96
|
-
scored_transactions = config.bank_reconciliation.chart_of_accounts.map do |coa|
|
97
|
-
{
|
98
|
-
coa: coa,
|
99
|
-
score: compare(coa.narration, transaction.narration)
|
100
|
-
}
|
101
|
-
end
|
102
|
-
|
103
|
-
scored_transactions.sort_by! { |t| t[:score] }.reverse!
|
104
|
-
|
105
|
-
best = scored_transactions.first
|
106
|
-
|
107
|
-
return nil unless best
|
108
|
-
return nil if best[:score] < score_threshold
|
109
|
-
|
110
|
-
coa = best[:coa]
|
111
|
-
|
112
|
-
transaction.coa_match_type = match_type
|
113
|
-
transaction.coa_code = coa.code
|
114
|
-
transaction
|
115
|
-
end
|
116
|
-
|
117
|
-
def compare(text1, text2)
|
118
|
-
text1_trigs = trigramify(text1)
|
119
|
-
text2_trigs = trigramify(text2)
|
120
|
-
|
121
|
-
all_cnt = (text1_trigs | text2_trigs).size
|
122
|
-
same_cnt = (text1_trigs & text2_trigs).size
|
123
|
-
|
124
|
-
same_cnt.to_f / all_cnt
|
125
|
-
end
|
126
|
-
|
127
|
-
def trigramify(text)
|
128
|
-
trigs = []
|
129
|
-
text.chars.each_cons(3) { |v| trigs << v.join }
|
130
|
-
trigs
|
131
|
-
end
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
@@ -1,90 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Clean
|
7
|
-
# Read transactions from a CSV file
|
8
|
-
class ReadTransactions
|
9
|
-
attr_reader :platform
|
10
|
-
attr_reader :transactions
|
11
|
-
|
12
|
-
def initialize(file)
|
13
|
-
@file = file
|
14
|
-
end
|
15
|
-
|
16
|
-
def read
|
17
|
-
csv_lines = File.read(@file).lines
|
18
|
-
|
19
|
-
@platform = detect_platform(csv_lines)
|
20
|
-
|
21
|
-
case platform
|
22
|
-
when :bankwest
|
23
|
-
read_bankwest(csv_lines)
|
24
|
-
when :bankwest2
|
25
|
-
read_bankwest2(csv_lines)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
private
|
30
|
-
|
31
|
-
def read_bankwest(csv_lines)
|
32
|
-
@transactions = []
|
33
|
-
|
34
|
-
# Skip the header line and parse each subsequent line
|
35
|
-
CSV.parse(csv_lines.join, headers: true).each do |row|
|
36
|
-
transaction = Models::Transaction.new(
|
37
|
-
bsb_number: row['BSB Number'],
|
38
|
-
account_number: row['Account Number'],
|
39
|
-
transaction_date: row['Transaction Date'],
|
40
|
-
narration: row['Narration'],
|
41
|
-
cheque_number: row['Cheque Number'],
|
42
|
-
debit: row['Debit'],
|
43
|
-
credit: row['Credit'],
|
44
|
-
balance: row['Balance'],
|
45
|
-
transaction_type: row['Transaction Type']
|
46
|
-
)
|
47
|
-
transaction.add_source_file(@file)
|
48
|
-
@transactions << transaction
|
49
|
-
end
|
50
|
-
|
51
|
-
@transactions
|
52
|
-
end
|
53
|
-
|
54
|
-
def read_bankwest2(csv_lines)
|
55
|
-
@transactions = []
|
56
|
-
|
57
|
-
# Skip the header line and parse each subsequent line
|
58
|
-
CSV.parse(csv_lines.join, headers: true).each do |row|
|
59
|
-
transaction = Models::Transaction.new(
|
60
|
-
bsb_number: row['BSB / Account Number'].split(' - ').first,
|
61
|
-
account_number: row['BSB / Account Number'].split(' - ').last,
|
62
|
-
transaction_date: row['Transaction Date'],
|
63
|
-
narration: row['Narration'],
|
64
|
-
cheque_number: row['Cheque Number'],
|
65
|
-
debit: row['Debit'],
|
66
|
-
credit: row['Credit'],
|
67
|
-
balance: row['Balance'],
|
68
|
-
transaction_type: row['Transaction Type']
|
69
|
-
)
|
70
|
-
transaction.add_source_file(@file)
|
71
|
-
@transactions << transaction
|
72
|
-
end
|
73
|
-
|
74
|
-
@transactions
|
75
|
-
end
|
76
|
-
|
77
|
-
# For bankwest the first row is the CSV will look like:
|
78
|
-
# BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type
|
79
|
-
def detect_platform(csv_lines)
|
80
|
-
return :bankwest if csv_lines.first.start_with?('BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
|
81
|
-
return :bankwest2 if csv_lines.first.start_with?('Account Name,BSB / Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
|
82
|
-
|
83
|
-
puts "Unknown platform detected. CSV columns are: #{csv_lines.first.strip}"
|
84
|
-
raise Appydave::Tools::Error, 'Unknown platform'
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
@@ -1,101 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Models
|
7
|
-
# Unified transaction model for raw and reconciled data
|
8
|
-
class Transaction
|
9
|
-
attr_accessor :bsb_number,
|
10
|
-
:account_number,
|
11
|
-
:transaction_date,
|
12
|
-
:narration,
|
13
|
-
:cheque_number,
|
14
|
-
:debit,
|
15
|
-
:credit,
|
16
|
-
:balance,
|
17
|
-
:transaction_type,
|
18
|
-
:platform,
|
19
|
-
:coa_code,
|
20
|
-
:coa_match_type,
|
21
|
-
:account_name,
|
22
|
-
:source_files
|
23
|
-
|
24
|
-
def initialize(bsb_number: nil,
|
25
|
-
account_number: nil,
|
26
|
-
transaction_date: nil,
|
27
|
-
narration: nil,
|
28
|
-
cheque_number: nil,
|
29
|
-
debit: nil,
|
30
|
-
credit: nil,
|
31
|
-
balance: nil,
|
32
|
-
transaction_type: nil,
|
33
|
-
platform: nil,
|
34
|
-
coa_code: nil,
|
35
|
-
coa_match_type: nil,
|
36
|
-
account_name: nil)
|
37
|
-
@bsb_number = bsb_number&.strip
|
38
|
-
@account_number = account_number&.strip
|
39
|
-
@transaction_date = transaction_date&.strip
|
40
|
-
@transaction_date = Date.strptime(@transaction_date, '%d/%m/%Y')
|
41
|
-
@narration = narration&.gsub(/\s{2,}/, ' ')&.strip
|
42
|
-
@cheque_number = cheque_number&.strip
|
43
|
-
@debit = debit&.strip
|
44
|
-
@credit = credit&.strip
|
45
|
-
@balance = balance&.strip
|
46
|
-
@transaction_type = transaction_type&.strip
|
47
|
-
@platform = platform
|
48
|
-
@coa_code = coa_code
|
49
|
-
@coa_match_type = coa_match_type
|
50
|
-
@account_name = account_name
|
51
|
-
@source_files = []
|
52
|
-
end
|
53
|
-
|
54
|
-
def add_source_file(source_file)
|
55
|
-
@source_files << source_file.strip unless @source_files.include?(source_file.strip)
|
56
|
-
end
|
57
|
-
|
58
|
-
# cheque_number
|
59
|
-
|
60
|
-
def self.csv_headers
|
61
|
-
%i[
|
62
|
-
platform
|
63
|
-
account_name
|
64
|
-
bsb_number
|
65
|
-
account_number
|
66
|
-
transaction_date
|
67
|
-
narration
|
68
|
-
debit
|
69
|
-
credit
|
70
|
-
balance
|
71
|
-
transaction_type
|
72
|
-
coa_code
|
73
|
-
coa_match_type
|
74
|
-
source_files
|
75
|
-
]
|
76
|
-
end
|
77
|
-
|
78
|
-
# @cheque_number,
|
79
|
-
|
80
|
-
def to_csv_row
|
81
|
-
[
|
82
|
-
@platform,
|
83
|
-
@account_name,
|
84
|
-
@bsb_number,
|
85
|
-
@account_number,
|
86
|
-
@transaction_date,
|
87
|
-
@narration,
|
88
|
-
@debit,
|
89
|
-
@credit,
|
90
|
-
@balance,
|
91
|
-
@transaction_type,
|
92
|
-
@coa_code,
|
93
|
-
@coa_match_type,
|
94
|
-
@source_files
|
95
|
-
]
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
@@ -1,97 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module Configuration
|
6
|
-
module Models
|
7
|
-
# Bank reconciliation configuration
|
8
|
-
class BankReconciliationConfig < ConfigBase
|
9
|
-
# def
|
10
|
-
# Retrieve all bank accounts
|
11
|
-
def bank_accounts
|
12
|
-
data['bank_accounts'].map do |account|
|
13
|
-
BankAccount.new(account)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
def chart_of_accounts
|
18
|
-
data['chart_of_accounts'].map do |entry|
|
19
|
-
ChartOfAccount.new(entry)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
def get_bank_account(account_number, bsb = nil)
|
24
|
-
account_data = data['bank_accounts'].find do |account|
|
25
|
-
account['account_number'] == account_number && (account['bsb'].nil? || account['bsb'] == bsb)
|
26
|
-
end
|
27
|
-
|
28
|
-
BankAccount.new(account_data) if account_data
|
29
|
-
end
|
30
|
-
|
31
|
-
# Retrieve a chart of account entry by code
|
32
|
-
def get_chart_of_account(code)
|
33
|
-
entry_data = data['chart_of_accounts'].find { |entry| entry['code'] == code }
|
34
|
-
ChartOfAccount.new(entry_data) if entry_data
|
35
|
-
end
|
36
|
-
|
37
|
-
def print
|
38
|
-
log.subheading 'Bank Reconciliation - Accounts'
|
39
|
-
|
40
|
-
tp bank_accounts, :account_number, :bsb, :name, :bank
|
41
|
-
|
42
|
-
log.subheading 'Bank Reconciliation - Chart of Accounts'
|
43
|
-
|
44
|
-
tp chart_of_accounts, :code, :narration
|
45
|
-
end
|
46
|
-
|
47
|
-
private
|
48
|
-
|
49
|
-
def default_data
|
50
|
-
{
|
51
|
-
'bank_accounts' => [],
|
52
|
-
'chart_of_accounts' => []
|
53
|
-
}
|
54
|
-
end
|
55
|
-
|
56
|
-
# Inner class to represent a bank account
|
57
|
-
class BankAccount
|
58
|
-
attr_accessor :account_number, :bsb, :name, :platform
|
59
|
-
|
60
|
-
def initialize(data)
|
61
|
-
@account_number = data['account_number']
|
62
|
-
@bsb = data['bsb']
|
63
|
-
@name = data['name']
|
64
|
-
@platform = data['platform']
|
65
|
-
end
|
66
|
-
|
67
|
-
def to_h
|
68
|
-
{
|
69
|
-
'account_number' => @account_number,
|
70
|
-
'bsb' => @bsb,
|
71
|
-
'name' => @name,
|
72
|
-
'platform' => @platform
|
73
|
-
}
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
# Inner class to represent a chart of account entry
|
78
|
-
class ChartOfAccount
|
79
|
-
attr_accessor :code, :narration
|
80
|
-
|
81
|
-
def initialize(data)
|
82
|
-
@code = data['code']
|
83
|
-
@narration = data['narration']
|
84
|
-
end
|
85
|
-
|
86
|
-
def to_h
|
87
|
-
{
|
88
|
-
'code' => @code,
|
89
|
-
'narration' => @narration
|
90
|
-
}
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|