appydave-tools 0.10.3 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/CHANGELOG.md +20 -0
- data/a.json +60 -0
- data/bin/bank_reconciliation.rb +23 -0
- data/bin/gpt_context.rb +37 -30
- data/lib/appydave/tools/gpt_context/file_collector.rb +43 -15
- data/lib/appydave/tools/gpt_context/options.rb +28 -0
- data/lib/appydave/tools/gpt_context/output_handler.rb +36 -0
- data/lib/appydave/tools/{prompt_tools → llm}/models/llm_info.rb +1 -1
- data/lib/appydave/tools/llm/openai_completion.rb +11 -0
- data/lib/appydave/tools/prompt_tools/prompt_completion.rb +2 -2
- data/lib/appydave/tools/version.rb +1 -1
- data/lib/appydave/tools.rb +11 -6
- data/package-lock.json +2 -2
- data/package.json +1 -1
- metadata +7 -9
- data/lib/appydave/tools/bank_reconciliation/_doc.md +0 -36
- data/lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb +0 -145
- data/lib/appydave/tools/bank_reconciliation/clean/mapper.rb +0 -136
- data/lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb +0 -90
- data/lib/appydave/tools/bank_reconciliation/models/transaction.rb +0 -101
- data/lib/appydave/tools/configuration/models/bank_reconciliation_config.rb +0 -97
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 88fcb99717ec45c2efcd0884772e1325a7de9cdca64c58c7e54e8c91203b333d
|
4
|
+
data.tar.gz: d29c8848556ef6c8a189fb3db0b6ad546a0e112d3dc5abffadf3f8e3bbaf058b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a1d033506927b381145b9cd57400e4b6ced89bfee1aeee9bbcf7977b9015bfebfc3116b1d86e1e8de8a185b46180754dabb7bf24b1d4c589e6887fef1db054a8
|
7
|
+
data.tar.gz: abedf296c032a813c0e1abb2bad7eb00b4d4f32c2ee19eda9623648a8a489f6085307f4dbb9763f526d03d86714314d1f613b6c62f848a80ed2c409b64cef5f1
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,23 @@
|
|
1
|
+
## [0.10.4](https://github.com/klueless-io/appydave-tools/compare/v0.10.3...v0.10.4) (2024-10-09)
|
2
|
+
|
3
|
+
|
4
|
+
### Bug Fixes
|
5
|
+
|
6
|
+
* cleanup code base before building documenation ([e473c44](https://github.com/klueless-io/appydave-tools/commit/e473c44546d8f0f5a35461fae263c348e7e2c58f))
|
7
|
+
* cleanup code base before building documenation ([38d8fb4](https://github.com/klueless-io/appydave-tools/commit/38d8fb46929e7c055308aa0b918c18a56fcd5842))
|
8
|
+
* cleanup code base before building documenation ([8dad12a](https://github.com/klueless-io/appydave-tools/commit/8dad12aadb9952f0174df45caed91c9e96070294))
|
9
|
+
* cleanup code base before building documenation ([eddfcc7](https://github.com/klueless-io/appydave-tools/commit/eddfcc78e39c93bfd4de6482690cc66cf90cc54a))
|
10
|
+
* extend prompt tools ([a628d08](https://github.com/klueless-io/appydave-tools/commit/a628d08fcdda521b7d148f0bab3261b879fca07d))
|
11
|
+
* extending bank reconciliation ([b435a20](https://github.com/klueless-io/appydave-tools/commit/b435a20237ef40a8a7d0a4f365ccd3eafdfcce1a))
|
12
|
+
* extending bank reconciliation ([b7878df](https://github.com/klueless-io/appydave-tools/commit/b7878dff0bca4ff1a3af7db19c280a8c245b6c3b))
|
13
|
+
|
14
|
+
## [0.10.3](https://github.com/klueless-io/appydave-tools/compare/v0.10.2...v0.10.3) (2024-06-17)
|
15
|
+
|
16
|
+
|
17
|
+
### Bug Fixes
|
18
|
+
|
19
|
+
* extending bank reconciliation with platform and banking mapping ([848b044](https://github.com/klueless-io/appydave-tools/commit/848b044bf4bb7c27bae6cf33aba400ab68eb105c))
|
20
|
+
|
1
21
|
## [0.10.2](https://github.com/klueless-io/appydave-tools/compare/v0.10.1...v0.10.2) (2024-06-17)
|
2
22
|
|
3
23
|
|
data/a.json
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
{
|
2
|
+
"tree": {
|
3
|
+
"bin": {
|
4
|
+
"gpt_context.rb": {
|
5
|
+
}
|
6
|
+
},
|
7
|
+
"lib": {
|
8
|
+
"appydave": {
|
9
|
+
"tools": {
|
10
|
+
"gpt_context": {
|
11
|
+
"file_collector.rb": {
|
12
|
+
},
|
13
|
+
"options.rb": {
|
14
|
+
},
|
15
|
+
"output_handler.rb": {
|
16
|
+
}
|
17
|
+
}
|
18
|
+
}
|
19
|
+
}
|
20
|
+
},
|
21
|
+
"spec": {
|
22
|
+
"appydave": {
|
23
|
+
"tools": {
|
24
|
+
"gpt_context": {
|
25
|
+
"file_collector_spec.rb": {
|
26
|
+
},
|
27
|
+
"output_handler_spec.rb": {
|
28
|
+
}
|
29
|
+
}
|
30
|
+
}
|
31
|
+
}
|
32
|
+
}
|
33
|
+
},
|
34
|
+
"content": [
|
35
|
+
{
|
36
|
+
"file": "bin/gpt_context.rb",
|
37
|
+
"content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# GPT Chats:\n# https://chatgpt.com/c/670df475-04f4-8002-a758-f5711bf433eb\n\n# Usage:\n# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*.rb'\n# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*' -e 'node_modules/**/*' -e 'package-lock.json' -e 'lib/openai_101/tools/prompts/*.txt'\n#\n# Get GPT Context Gatherer code\n# ./bin/gpt_context.rb -i 'bin/**/*gather*.rb' -i 'lib/openai_101/tools/**/*gather*.rb'\n$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))\n\nrequire 'appydave/tools'\nrequire 'appydave/tools/gpt_context/options'\nrequire 'json'\nrequire 'optparse'\nrequire 'clipboard'\nrequire 'pathname'\n\noptions = Appydave::Tools::GptContext::Options.new(\n working_directory: nil\n)\n\nOptionParser.new do |opts|\n opts.banner = 'Usage: gather_content.rb [options]'\n\n opts.on('-i', '--include PATTERN', 'Pattern or file to include (can be used multiple times)') do |pattern|\n options.include_patterns << pattern\n end\n\n opts.on('-e', '--exclude PATTERN', 'Pattern or file to exclude (can be used multiple times)') do |pattern|\n options.exclude_patterns << pattern\n end\n\n opts.on('-f', '--format FORMAT', 'Output format: content, tree, or json, if not provided then both are used') do |format|\n options.format = format\n end\n\n opts.on('-l', '--line-limit LIMIT', 'Limit the number of lines included from each file') do |limit|\n options.line_limit = limit.to_i\n end\n\n # New option for specifying base directory\n opts.on('-b', '--base-dir DIRECTORY', 'Set the base directory to gather files from') do |directory|\n options.working_directory = directory\n end\n\n # Debug output options\n opts.on('-d', '--debug [MODE]', 'Enable debug mode [none, info, params, debug]', 'none', 'info', 'params', 'debug') do |debug|\n options.debug = debug || 'info'\n end\n\n # Output targets: clipboard or file\n opts.on('-o', '--output TARGET', 'Output target: clipboard, or a file path (can be used multiple times)') do |target|\n options.output_target << target\n end\n\n opts.on_tail('-h', '--help', 'Show this message') do\n puts opts\n puts \"\\nExamples:\"\n puts \" #{File.basename($PROGRAM_NAME)} -i 'lib/**/*.rb' -e 'lib/excluded/**/*.rb' -d\"\n puts \" #{File.basename($PROGRAM_NAME)} --include 'src/**/*.js' --exclude 'src/vendor/**/*.js'\"\n\n puts ''\n puts ' # Get GPT Context Gatherer code that is found in any folder (bin, lib & spec)'\n puts \" #{File.basename($PROGRAM_NAME)} -i '**/*gather*.rb'\"\n exit\n end\nend.parse!\n\nif options.include_patterns.empty? && options.exclude_patterns.empty? && options.format.nil?\n script_name = File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME))\n\n puts 'No options provided to GPT Context. Please specify patterns to include or exclude.'\n puts \"For help, run: #{script_name} --help\"\n exit\nend\n\nif options.output_target.empty?\n puts 'No output target provided. Will default to `clipboard`. You can set the output target using -o'\n options.output_target << 'clipboard'\nend\n\npp options if options.debug == 'params'\n\noptions.working_directory ||= Dir.pwd\n\ngatherer = Appydave::Tools::GptContext::FileCollector.new(options)\ncontent = gatherer.build\n\nif %w[info debug].include?(options.debug)\n puts '-' * 80\n puts content\n puts '-' * 80\nend\n\noutput_handler = Appydave::Tools::GptContext::OutputHandler.new(content, options)\noutput_handler.execute\n\npp options if options.debug == 'debug'\n"
|
38
|
+
},
|
39
|
+
{
|
40
|
+
"file": "lib/appydave/tools/gpt_context/file_collector.rb",
|
41
|
+
"content": "# frozen_string_literal: true\n\nmodule Appydave\n module Tools\n # Build GPT context from various sources\n module GptContext\n # Gathers file names and content based on include and exclude patterns\n class FileCollector\n def initialize(options)\n @options = options\n @include_patterns = options.include_patterns\n @exclude_patterns = options.exclude_patterns\n @format = options.format\n @working_directory = options.working_directory\n @line_limit = options.line_limit\n end\n\n def build\n FileUtils.cd(@working_directory) if @working_directory && Dir.exist?(@working_directory)\n\n formats = @format.split(',')\n result = formats.map do |fmt|\n case fmt\n when 'tree'\n build_tree\n when 'content'\n build_content\n when 'json'\n build_json\n else\n ''\n end\n end.join(\"\\n\\n\")\n\n FileUtils.cd(Dir.home) if @working_directory\n\n result\n end\n\n private\n\n def build_content\n concatenated_content = []\n\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path) || File.directory?(file_path)\n\n content = \"# file: #{file_path}\\n\\n#{read_file_content(file_path)}\"\n concatenated_content << content\n end\n end\n\n concatenated_content.join(\"\\n\\n\")\n end\n\n def read_file_content(file_path)\n lines = File.readlines(file_path)\n return lines.first(@line_limit).join if @line_limit\n\n lines.join\n end\n\n def build_tree\n tree_view = {}\n\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path)\n\n path_parts = file_path.split('/')\n insert_into_tree(tree_view, path_parts)\n end\n end\n\n build_tree_pretty(tree_view).rstrip\n end\n\n def insert_into_tree(tree, path_parts)\n node = tree\n path_parts.each do |part|\n node[part] ||= {}\n node = node[part]\n end\n end\n\n def build_tree_pretty(node, prefix: '', is_last: true, output: ''.dup)\n node.each_with_index do |(part, child), index|\n connector = is_last && index == node.size - 1 ? '└' : '├'\n output << \"#{prefix}#{connector}─ #{part}\\n\"\n next_prefix = is_last && index == node.size - 1 ? ' ' : '│ '\n build_tree_pretty(child, prefix: \"#{prefix}#{next_prefix}\", is_last: child.empty? || index == node.size - 1, output: output)\n end\n output\n end\n\n def build_json\n json_output = {\n 'tree' => {},\n 'content' => []\n }\n\n # Building tree structure in JSON\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path)\n\n path_parts = file_path.split('/')\n insert_into_tree(json_output['tree'], path_parts)\n end\n end\n\n # Building content structure in JSON\n @include_patterns.each do |pattern|\n Dir.glob(pattern).each do |file_path|\n next if excluded?(file_path) || File.directory?(file_path)\n\n json_output['content'] << {\n 'file' => file_path,\n 'content' => read_file_content(file_path)\n }\n end\n end\n\n JSON.pretty_generate(json_output)\n end\n\n def excluded?(file_path)\n @exclude_patterns.any? { |pattern| File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) }\n end\n end\n end\n end\nend\n"
|
42
|
+
},
|
43
|
+
{
|
44
|
+
"file": "lib/appydave/tools/gpt_context/options.rb",
|
45
|
+
"content": "# frozen_string_literal: true\n\nmodule Appydave\n module Tools\n module GptContext\n # Struct with keyword_init: true to allow named parameters\n Options = Struct.new(\n :include_patterns,\n :exclude_patterns,\n :format,\n :line_limit,\n :debug,\n :output_target,\n :working_directory,\n keyword_init: true\n ) do\n def initialize(**args)\n super\n self.include_patterns ||= []\n self.exclude_patterns ||= []\n self.format ||= 'tree,content'\n self.debug ||= 'none'\n self.output_target ||= []\n end\n end\n end\n end\nend\n"
|
46
|
+
},
|
47
|
+
{
|
48
|
+
"file": "lib/appydave/tools/gpt_context/output_handler.rb",
|
49
|
+
"content": "# frozen_string_literal: true\n\nmodule Appydave\n module Tools\n module GptContext\n class OutputHandler\n def initialize(content, options)\n @content = content\n @output_targets = options.output_target\n @working_directory = options.working_directory\n end\n\n def execute\n @output_targets.each do |target|\n case target\n when 'clipboard'\n Clipboard.copy(@content)\n when /^.+$/\n write_to_file(target)\n end\n end\n end\n\n private\n\n attr_reader :content, :output_targets, :working_directory\n\n def write_to_file(target)\n resolved_path = Pathname.new(target).absolute? ? target : File.join(working_directory, target)\n File.write(resolved_path, content)\n end\n end\n end\n end\nend\n"
|
50
|
+
},
|
51
|
+
{
|
52
|
+
"file": "spec/appydave/tools/gpt_context/file_collector_spec.rb",
|
53
|
+
"content": "# frozen_string_literal: true\n\nRSpec.describe Appydave::Tools::GptContext::FileCollector do\n describe '#build' do\n subject { described_class.new(include_patterns: include_patterns, exclude_patterns: exclude_patterns, format: format, line_limit: line_limit) }\n\n let(:include_patterns) { ['spec/fixtures/gpt-content-gatherer/**/*.txt'] }\n let(:exclude_patterns) { ['spec/fixtures/gpt-content-gatherer/excluded/*.txt', '**/deep/**/*'] }\n let(:format) { 'content' }\n let(:line_limit) { nil }\n\n context 'when gathering content' do\n it 'concatenates content from files matching include patterns' do\n expect(subject.build).to include('File 1 content', 'File 2 content')\n end\n\n it 'excludes content from files matching exclude patterns' do\n expect(subject.build).not_to include('Excluded file content')\n expect(subject.build).not_to include('Deep 1')\n expect(subject.build).not_to include('Deep 1')\n end\n\n it 'includes file paths as headers in the gathered content' do\n expect(subject.build)\n .to include('# file: spec/fixtures/gpt-content-gatherer/included/file1.txt')\n .and include('# file: spec/fixtures/gpt-content-gatherer/included/file2.txt')\n end\n end\n\n context 'when line limit is set' do\n let(:line_limit) { 1 }\n\n it 'limits the number of lines included from each file' do\n expect(subject.build).not_to include('Line #2')\n end\n end\n end\n\n describe '#build with tree format' do\n subject { described_class.new(include_patterns: include_patterns, exclude_patterns: exclude_patterns, format: 'tree') }\n\n let(:include_patterns) { ['spec/fixtures/gpt-content-gatherer/**/*'] }\n let(:exclude_patterns) { [] }\n\n before do\n allow(Dir).to receive(:glob).and_return(\n [\n 'spec/fixtures/gpt-content-gatherer/included/file1.txt',\n 'spec/fixtures/gpt-content-gatherer/included/file2.txt',\n 'spec/fixtures/gpt-content-gatherer/included/subdir/file3.txt'\n ]\n )\n allow(File).to receive(:directory?).and_return(false)\n end\n\n it 'prints a tree view of the included files and directories with improved ASCII art' do\n expected_output = <<~TREE\n └─ spec\n └─ fixtures\n └─ gpt-content-gatherer\n └─ included\n ├─ file1.txt\n ├─ file2.txt\n └─ subdir\n └─ file3.txt\n TREE\n\n expect(subject.build.strip).to eq(expected_output.strip)\n end\n end\n\n describe '#build with both formats' do\n subject { described_class.new(include_patterns: include_patterns, exclude_patterns: exclude_patterns, format: 'tree,content') }\n\n let(:include_patterns) { ['spec/fixtures/gpt-content-gatherer/**/*'] }\n let(:exclude_patterns) { [] }\n\n before do\n allow(Dir).to receive(:glob).and_return(\n [\n 'spec/fixtures/gpt-content-gatherer/included/file1.txt',\n 'spec/fixtures/gpt-content-gatherer/included/file2.txt',\n 'spec/fixtures/gpt-content-gatherer/included/subdir/file3.txt'\n ]\n )\n allow(File).to receive(:directory?).and_return(false)\n end\n\n it 'prints both a tree view and the file contents' do\n expected_output_tree = <<~TREE\n └─ spec\n └─ fixtures\n └─ gpt-content-gatherer\n └─ included\n ├─ file1.txt\n ├─ file2.txt\n └─ subdir\n └─ file3.txt\n TREE\n\n expected_output_content = <<~CONTENT\n # file: spec/fixtures/gpt-content-gatherer/included/file1.txt\n\n File 1 content\n Line #2\n\n # file: spec/fixtures/gpt-content-gatherer/included/file2.txt\n\n File 2 content\n Line #2\n\n # file: spec/fixtures/gpt-content-gatherer/included/subdir/file3.txt\n\n File 3 content\n Line #2\n CONTENT\n\n expect(subject.build).to include(expected_output_tree.strip)\n expect(subject.build).to include(expected_output_content.strip)\n end\n end\nend\n"
|
54
|
+
},
|
55
|
+
{
|
56
|
+
"file": "spec/appydave/tools/gpt_context/output_handler_spec.rb",
|
57
|
+
"content": "# frozen_string_literal: true\n\nRSpec.describe Appydave::Tools::GptContext::OutputHandler do\n subject { described_class.new(content, options) }\n\n let(:content) { 'Sample content' }\n let(:options) { double('Options', output_target: ['clipboard'], working_directory: Dir.pwd) }\n\n describe '#execute' do\n it 'copies content to clipboard when output target is clipboard' do\n expect(Clipboard).to receive(:copy).with(content)\n subject.execute\n end\n\n context 'when output target is a file' do\n let(:options) { double('Options', output_target: ['output.txt'], working_directory: Dir.pwd) }\n\n it 'writes content to the specified file' do\n expect(File).to receive(:write).with(File.join(Dir.pwd, 'output.txt'), content)\n subject.execute\n end\n end\n\n context 'when multiple output targets are specified' do\n let(:options) { double('Options', output_target: ['clipboard', 'output.txt'], working_directory: Dir.pwd) }\n\n it 'copies content to clipboard and writes to file' do\n expect(Clipboard).to receive(:copy).with(content)\n expect(File).to receive(:write).with(File.join(Dir.pwd, 'output.txt'), content)\n subject.execute\n end\n end\n end\nend\n"
|
58
|
+
}
|
59
|
+
]
|
60
|
+
}
|
data/bin/bank_reconciliation.rb
CHANGED
@@ -14,6 +14,7 @@ class BankReconciliationCLI
|
|
14
14
|
def initialize
|
15
15
|
@commands = {
|
16
16
|
'clean' => method(:clean_transactions),
|
17
|
+
'transform' => method(:transform),
|
17
18
|
'process' => method(:process_transactions),
|
18
19
|
'filter' => method(:filter_transactions)
|
19
20
|
}
|
@@ -73,6 +74,28 @@ class BankReconciliationCLI
|
|
73
74
|
cleaner.clean_transactions(include_patterns, output_file)
|
74
75
|
end
|
75
76
|
|
77
|
+
def transform(args)
|
78
|
+
options = {}
|
79
|
+
OptionParser.new do |opts|
|
80
|
+
opts.banner = 'Usage: bank_reconciliation.rb clean [options]'
|
81
|
+
|
82
|
+
opts.on('-c', '--to-csv', 'Write chart of accounts JSON to CSV') { options[:to_csv] = true }
|
83
|
+
opts.on('-j', '--to-json', 'Write chart of accounts CSV to JSON') { options[:to_json] = true }
|
84
|
+
|
85
|
+
opts.on('-d', '--debug', 'Enable debug mode') do
|
86
|
+
options[:debug] = true
|
87
|
+
end
|
88
|
+
|
89
|
+
opts.on_tail('-h', '--help', 'Show this message') do
|
90
|
+
puts opts
|
91
|
+
exit
|
92
|
+
end
|
93
|
+
end.parse!(args)
|
94
|
+
|
95
|
+
Appydave::Tools::Configuration::Models::BankReconciliationConfig.new.coa_to_csv if options[:to_csv]
|
96
|
+
Appydave::Tools::Configuration::Models::BankReconciliationConfig.new.coa_csv_to_json if options[:to_json]
|
97
|
+
end
|
98
|
+
|
76
99
|
def process_transactions(args)
|
77
100
|
options = {}
|
78
101
|
OptionParser.new do |opts|
|
data/bin/gpt_context.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
|
+
# GPT Chats:
|
5
|
+
# https://chatgpt.com/c/670df475-04f4-8002-a758-f5711bf433eb
|
6
|
+
|
4
7
|
# Usage:
|
5
8
|
# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*.rb'
|
6
9
|
# ./bin/gpt_context.rb -d -i 'lib/openai_101/tools/**/*' -e 'node_modules/**/*' -e 'package-lock.json' -e 'lib/openai_101/tools/prompts/*.txt'
|
@@ -11,39 +14,42 @@ $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
|
11
14
|
|
12
15
|
require 'appydave/tools'
|
13
16
|
|
14
|
-
options =
|
15
|
-
|
16
|
-
|
17
|
-
format: 'tree,content',
|
18
|
-
line_limit: nil,
|
19
|
-
debug: 'none'
|
20
|
-
}
|
17
|
+
options = Appydave::Tools::GptContext::Options.new(
|
18
|
+
working_directory: nil
|
19
|
+
)
|
21
20
|
|
22
21
|
OptionParser.new do |opts|
|
23
22
|
opts.banner = 'Usage: gather_content.rb [options]'
|
24
23
|
|
25
24
|
opts.on('-i', '--include PATTERN', 'Pattern or file to include (can be used multiple times)') do |pattern|
|
26
|
-
options
|
25
|
+
options.include_patterns << pattern
|
27
26
|
end
|
28
27
|
|
29
28
|
opts.on('-e', '--exclude PATTERN', 'Pattern or file to exclude (can be used multiple times)') do |pattern|
|
30
|
-
options
|
29
|
+
options.exclude_patterns << pattern
|
31
30
|
end
|
32
31
|
|
33
|
-
opts.on('-f', '--format FORMAT', 'Output format: content or
|
34
|
-
options
|
32
|
+
opts.on('-f', '--format FORMAT', 'Output format: content, tree, or json, if not provided then both are used') do |format|
|
33
|
+
options.format = format
|
35
34
|
end
|
36
35
|
|
37
36
|
opts.on('-l', '--line-limit LIMIT', 'Limit the number of lines included from each file') do |limit|
|
38
|
-
options
|
37
|
+
options.line_limit = limit.to_i
|
39
38
|
end
|
40
39
|
|
41
|
-
#
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
40
|
+
# New option for specifying base directory
|
41
|
+
opts.on('-b', '--base-dir DIRECTORY', 'Set the base directory to gather files from') do |directory|
|
42
|
+
options.working_directory = directory
|
43
|
+
end
|
44
|
+
|
45
|
+
# Debug output options
|
46
|
+
opts.on('-d', '--debug [MODE]', 'Enable debug mode [none, info, params, debug]', 'none', 'info', 'params', 'debug') do |debug|
|
47
|
+
options.debug = debug || 'info'
|
48
|
+
end
|
49
|
+
|
50
|
+
# Output targets: clipboard or file
|
51
|
+
opts.on('-o', '--output TARGET', 'Output target: clipboard, or a file path (can be used multiple times)') do |target|
|
52
|
+
options.output_target << target
|
47
53
|
end
|
48
54
|
|
49
55
|
opts.on_tail('-h', '--help', 'Show this message') do
|
@@ -59,7 +65,7 @@ OptionParser.new do |opts|
|
|
59
65
|
end
|
60
66
|
end.parse!
|
61
67
|
|
62
|
-
if options
|
68
|
+
if options.include_patterns.empty? && options.exclude_patterns.empty? && options.format.nil?
|
63
69
|
script_name = File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME))
|
64
70
|
|
65
71
|
puts 'No options provided to GPT Context. Please specify patterns to include or exclude.'
|
@@ -67,24 +73,25 @@ if options[:include_patterns].empty? && options[:exclude_patterns].empty? && opt
|
|
67
73
|
exit
|
68
74
|
end
|
69
75
|
|
70
|
-
|
76
|
+
if options.output_target.empty?
|
77
|
+
puts 'No output target provided. Will default to `clipboard`. You can set the output target using -o'
|
78
|
+
options.output_target << 'clipboard'
|
79
|
+
end
|
80
|
+
|
81
|
+
pp options if options.debug == 'params'
|
71
82
|
|
72
|
-
|
73
|
-
include_patterns: options[:include_patterns],
|
74
|
-
exclude_patterns: options[:exclude_patterns],
|
75
|
-
format: options[:format],
|
76
|
-
line_limit: options[:line_limit],
|
77
|
-
working_directory: Dir.pwd
|
78
|
-
)
|
83
|
+
options.working_directory ||= Dir.pwd
|
79
84
|
|
85
|
+
gatherer = Appydave::Tools::GptContext::FileCollector.new(options)
|
80
86
|
content = gatherer.build
|
81
87
|
|
82
|
-
if %w[
|
88
|
+
if %w[info debug].include?(options.debug)
|
83
89
|
puts '-' * 80
|
84
90
|
puts content
|
85
91
|
puts '-' * 80
|
86
92
|
end
|
87
93
|
|
88
|
-
|
94
|
+
output_handler = Appydave::Tools::GptContext::OutputHandler.new(content, options)
|
95
|
+
output_handler.execute
|
89
96
|
|
90
|
-
|
97
|
+
pp options if options.debug == 'debug'
|
@@ -6,32 +6,33 @@ module Appydave
|
|
6
6
|
module GptContext
|
7
7
|
# Gathers file names and content based on include and exclude patterns
|
8
8
|
class FileCollector
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
@
|
13
|
-
@
|
14
|
-
@
|
15
|
-
@
|
16
|
-
@line_limit = line_limit
|
9
|
+
def initialize(options)
|
10
|
+
@options = options
|
11
|
+
@include_patterns = options.include_patterns
|
12
|
+
@exclude_patterns = options.exclude_patterns
|
13
|
+
@format = options.format
|
14
|
+
@working_directory = options.working_directory
|
15
|
+
@line_limit = options.line_limit
|
17
16
|
end
|
18
17
|
|
19
18
|
def build
|
20
|
-
FileUtils.cd(working_directory) if working_directory && Dir.exist?(working_directory)
|
19
|
+
FileUtils.cd(@working_directory) if @working_directory && Dir.exist?(@working_directory)
|
21
20
|
|
22
|
-
formats = format.split(',')
|
21
|
+
formats = @format.split(',')
|
23
22
|
result = formats.map do |fmt|
|
24
23
|
case fmt
|
25
24
|
when 'tree'
|
26
25
|
build_tree
|
27
26
|
when 'content'
|
28
27
|
build_content
|
28
|
+
when 'json'
|
29
|
+
build_json
|
29
30
|
else
|
30
31
|
''
|
31
32
|
end
|
32
33
|
end.join("\n\n")
|
33
34
|
|
34
|
-
FileUtils.cd(Dir.home) if working_directory
|
35
|
+
FileUtils.cd(Dir.home) if @working_directory
|
35
36
|
|
36
37
|
result
|
37
38
|
end
|
@@ -41,7 +42,7 @@ module Appydave
|
|
41
42
|
def build_content
|
42
43
|
concatenated_content = []
|
43
44
|
|
44
|
-
include_patterns.each do |pattern|
|
45
|
+
@include_patterns.each do |pattern|
|
45
46
|
Dir.glob(pattern).each do |file_path|
|
46
47
|
next if excluded?(file_path) || File.directory?(file_path)
|
47
48
|
|
@@ -55,7 +56,7 @@ module Appydave
|
|
55
56
|
|
56
57
|
def read_file_content(file_path)
|
57
58
|
lines = File.readlines(file_path)
|
58
|
-
return lines.first(line_limit).join if line_limit
|
59
|
+
return lines.first(@line_limit).join if @line_limit
|
59
60
|
|
60
61
|
lines.join
|
61
62
|
end
|
@@ -63,7 +64,7 @@ module Appydave
|
|
63
64
|
def build_tree
|
64
65
|
tree_view = {}
|
65
66
|
|
66
|
-
include_patterns.each do |pattern|
|
67
|
+
@include_patterns.each do |pattern|
|
67
68
|
Dir.glob(pattern).each do |file_path|
|
68
69
|
next if excluded?(file_path)
|
69
70
|
|
@@ -93,8 +94,35 @@ module Appydave
|
|
93
94
|
output
|
94
95
|
end
|
95
96
|
|
97
|
+
def build_json
|
98
|
+
json_output = {
|
99
|
+
'tree' => {},
|
100
|
+
'content' => []
|
101
|
+
}
|
102
|
+
|
103
|
+
# Building tree structure in JSON
|
104
|
+
@include_patterns.each do |pattern|
|
105
|
+
Dir.glob(pattern).each do |file_path|
|
106
|
+
next if excluded?(file_path)
|
107
|
+
|
108
|
+
path_parts = file_path.split('/')
|
109
|
+
insert_into_tree(json_output['tree'], path_parts)
|
110
|
+
|
111
|
+
# Building content structure in JSON
|
112
|
+
next if excluded?(file_path) || File.directory?(file_path)
|
113
|
+
|
114
|
+
json_output['content'] << {
|
115
|
+
'file' => file_path,
|
116
|
+
'content' => read_file_content(file_path)
|
117
|
+
}
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
JSON.pretty_generate(json_output)
|
122
|
+
end
|
123
|
+
|
96
124
|
def excluded?(file_path)
|
97
|
-
exclude_patterns.any? { |pattern| File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) }
|
125
|
+
@exclude_patterns.any? { |pattern| File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) }
|
98
126
|
end
|
99
127
|
end
|
100
128
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Appydave
|
4
|
+
module Tools
|
5
|
+
module GptContext
|
6
|
+
# Struct with keyword_init: true to allow named parameters
|
7
|
+
Options = Struct.new(
|
8
|
+
:include_patterns,
|
9
|
+
:exclude_patterns,
|
10
|
+
:format,
|
11
|
+
:line_limit,
|
12
|
+
:debug,
|
13
|
+
:output_target,
|
14
|
+
:working_directory,
|
15
|
+
keyword_init: true
|
16
|
+
) do
|
17
|
+
def initialize(**args)
|
18
|
+
super
|
19
|
+
self.include_patterns ||= []
|
20
|
+
self.exclude_patterns ||= []
|
21
|
+
self.format ||= 'tree,content'
|
22
|
+
self.debug ||= 'none'
|
23
|
+
self.output_target ||= []
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Appydave
|
4
|
+
module Tools
|
5
|
+
module GptContext
|
6
|
+
# OutputHandler is responsible for writing the output to the desired target
|
7
|
+
class OutputHandler
|
8
|
+
def initialize(content, options)
|
9
|
+
@content = content
|
10
|
+
@output_targets = options.output_target
|
11
|
+
@working_directory = options.working_directory
|
12
|
+
end
|
13
|
+
|
14
|
+
def execute
|
15
|
+
@output_targets.each do |target|
|
16
|
+
case target
|
17
|
+
when 'clipboard'
|
18
|
+
Clipboard.copy(@content)
|
19
|
+
when /^.+$/
|
20
|
+
write_to_file(target)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
attr_reader :content, :output_targets, :working_directory
|
28
|
+
|
29
|
+
def write_to_file(target)
|
30
|
+
resolved_path = Pathname.new(target).absolute? ? target : File.join(working_directory, target)
|
31
|
+
File.write(resolved_path, content)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -16,7 +16,7 @@ module Appydave
|
|
16
16
|
attr_reader :clipboard
|
17
17
|
|
18
18
|
def initialize(options = {})
|
19
|
-
|
19
|
+
setup_options(options)
|
20
20
|
|
21
21
|
validate_options
|
22
22
|
end
|
@@ -44,7 +44,7 @@ module Appydave
|
|
44
44
|
|
45
45
|
private
|
46
46
|
|
47
|
-
def
|
47
|
+
def setup_options(options)
|
48
48
|
@prompt = options.delete(:prompt)
|
49
49
|
@prompt_file = options.delete(:prompt_file)
|
50
50
|
@llm = Appydave::Tools::PromptTools::Models::LlmInfo.new(
|
data/lib/appydave/tools.rb
CHANGED
@@ -7,6 +7,7 @@ require 'json'
|
|
7
7
|
require 'open3'
|
8
8
|
require 'openai'
|
9
9
|
require 'optparse'
|
10
|
+
require 'pathname'
|
10
11
|
require 'k_log'
|
11
12
|
require 'active_model'
|
12
13
|
|
@@ -24,6 +25,9 @@ require 'appydave/tools/types/hash_type'
|
|
24
25
|
require 'appydave/tools/types/array_type'
|
25
26
|
require 'appydave/tools/types/base_model'
|
26
27
|
|
28
|
+
require 'appydave/tools/llm/models/llm_info'
|
29
|
+
require 'appydave/tools/llm/openai_completion'
|
30
|
+
|
27
31
|
require 'appydave/tools/cli_actions/base_action'
|
28
32
|
|
29
33
|
# May want to move this into the tools location
|
@@ -31,23 +35,24 @@ require 'appydave/tools/cli_actions/prompt_completion_action'
|
|
31
35
|
require 'appydave/tools/cli_actions/get_video_action'
|
32
36
|
require 'appydave/tools/cli_actions/update_video_action'
|
33
37
|
|
38
|
+
require 'appydave/tools/gpt_context/options'
|
34
39
|
require 'appydave/tools/gpt_context/file_collector'
|
40
|
+
require 'appydave/tools/gpt_context/output_handler'
|
35
41
|
|
36
42
|
require 'appydave/tools/configuration/openai'
|
37
43
|
require 'appydave/tools/configuration/configurable'
|
38
44
|
require 'appydave/tools/configuration/config'
|
39
45
|
require 'appydave/tools/configuration/models/config_base'
|
40
46
|
require 'appydave/tools/configuration/models/settings_config'
|
41
|
-
require 'appydave/tools/configuration/models/bank_reconciliation_config'
|
47
|
+
# require 'appydave/tools/configuration/models/bank_reconciliation_config'
|
42
48
|
require 'appydave/tools/configuration/models/channels_config'
|
43
49
|
require 'appydave/tools/configuration/models/youtube_automation_config'
|
44
50
|
require 'appydave/tools/name_manager/project_name'
|
45
|
-
require 'appydave/tools/bank_reconciliation/clean/clean_transactions'
|
46
|
-
require 'appydave/tools/bank_reconciliation/clean/read_transactions'
|
47
|
-
require 'appydave/tools/bank_reconciliation/clean/mapper'
|
48
|
-
require 'appydave/tools/bank_reconciliation/models/transaction'
|
51
|
+
# require 'appydave/tools/bank_reconciliation/clean/clean_transactions'
|
52
|
+
# require 'appydave/tools/bank_reconciliation/clean/read_transactions'
|
53
|
+
# require 'appydave/tools/bank_reconciliation/clean/mapper'
|
54
|
+
# require 'appydave/tools/bank_reconciliation/models/transaction'
|
49
55
|
|
50
|
-
require 'appydave/tools/prompt_tools/models/llm_info'
|
51
56
|
require 'appydave/tools/prompt_tools/prompt_completion'
|
52
57
|
|
53
58
|
require 'appydave/tools/subtitle_master/clean'
|
data/package-lock.json
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
{
|
2
2
|
"name": "appydave-tools",
|
3
|
-
"version": "0.
|
3
|
+
"version": "0.11.0",
|
4
4
|
"lockfileVersion": 3,
|
5
5
|
"requires": true,
|
6
6
|
"packages": {
|
7
7
|
"": {
|
8
8
|
"name": "appydave-tools",
|
9
|
-
"version": "0.
|
9
|
+
"version": "0.11.0",
|
10
10
|
"devDependencies": {
|
11
11
|
"@klueless-js/semantic-release-rubygem": "github:klueless-js/semantic-release-rubygem",
|
12
12
|
"@semantic-release/changelog": "^6.0.3",
|
data/package.json
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: appydave-tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Cruwys
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-10-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|
@@ -156,6 +156,7 @@ files:
|
|
156
156
|
- LICENSE.txt
|
157
157
|
- README.md
|
158
158
|
- Rakefile
|
159
|
+
- a.json
|
159
160
|
- bin/bank_reconciliation.rb
|
160
161
|
- bin/configuration.rb
|
161
162
|
- bin/console
|
@@ -167,11 +168,6 @@ files:
|
|
167
168
|
- bin/youtube_manager.rb
|
168
169
|
- images.log
|
169
170
|
- lib/appydave/tools.rb
|
170
|
-
- lib/appydave/tools/bank_reconciliation/_doc.md
|
171
|
-
- lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb
|
172
|
-
- lib/appydave/tools/bank_reconciliation/clean/mapper.rb
|
173
|
-
- lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb
|
174
|
-
- lib/appydave/tools/bank_reconciliation/models/transaction.rb
|
175
171
|
- lib/appydave/tools/cli_actions/_doc.md
|
176
172
|
- lib/appydave/tools/cli_actions/base_action.rb
|
177
173
|
- lib/appydave/tools/cli_actions/get_video_action.rb
|
@@ -180,7 +176,6 @@ files:
|
|
180
176
|
- lib/appydave/tools/configuration/_doc.md
|
181
177
|
- lib/appydave/tools/configuration/config.rb
|
182
178
|
- lib/appydave/tools/configuration/configurable.rb
|
183
|
-
- lib/appydave/tools/configuration/models/bank_reconciliation_config.rb
|
184
179
|
- lib/appydave/tools/configuration/models/channels_config.rb
|
185
180
|
- lib/appydave/tools/configuration/models/config_base.rb
|
186
181
|
- lib/appydave/tools/configuration/models/settings_config copy.xrb
|
@@ -190,10 +185,13 @@ files:
|
|
190
185
|
- lib/appydave/tools/debuggable.rb
|
191
186
|
- lib/appydave/tools/gpt_context/_doc.md
|
192
187
|
- lib/appydave/tools/gpt_context/file_collector.rb
|
188
|
+
- lib/appydave/tools/gpt_context/options.rb
|
189
|
+
- lib/appydave/tools/gpt_context/output_handler.rb
|
190
|
+
- lib/appydave/tools/llm/models/llm_info.rb
|
191
|
+
- lib/appydave/tools/llm/openai_completion.rb
|
193
192
|
- lib/appydave/tools/name_manager/_doc.md
|
194
193
|
- lib/appydave/tools/name_manager/project_name.rb
|
195
194
|
- lib/appydave/tools/prompt_tools/_doc.md
|
196
|
-
- lib/appydave/tools/prompt_tools/models/llm_info.rb
|
197
195
|
- lib/appydave/tools/prompt_tools/prompt_completion.rb
|
198
196
|
- lib/appydave/tools/subtitle_master/_doc.md
|
199
197
|
- lib/appydave/tools/subtitle_master/clean.rb
|
@@ -1,36 +0,0 @@
|
|
1
|
-
# Bank reconciliation
|
2
|
-
|
3
|
-
[ChatGPT conversation](https://chatgpt.com/c/5d382562-95e5-4243-9b74-c3807d363486)
|
4
|
-
|
5
|
-
|
6
|
-
## Code structure
|
7
|
-
|
8
|
-
```bash
|
9
|
-
├─ lib
|
10
|
-
│ ├─ appydave
|
11
|
-
│ │ └─ tools
|
12
|
-
│ │ ├─ bank_reconciliation
|
13
|
-
│ │ │ ├─ clean
|
14
|
-
│ │ │ │ ├─ read_transactions.rb
|
15
|
-
│ │ │ │ ├─ transaction_cleaner.rb
|
16
|
-
│ │ │ ├─ models
|
17
|
-
│ │ │ │ ├─ raw_transaction.rb
|
18
|
-
│ │ │ │ └─ reconciled_transaction.rb
|
19
|
-
│ │ └─ configuration
|
20
|
-
│ │ └─ models
|
21
|
-
│ │ └─ bank_reconciliation_config.rb
|
22
|
-
└─ spec
|
23
|
-
├─ appydave
|
24
|
-
│ ├─ tools
|
25
|
-
│ │ ├─ bank_reconciliation
|
26
|
-
│ │ │ ├─ clean
|
27
|
-
│ │ │ │ ├─ read_transactions_spec.rb
|
28
|
-
│ │ │ ├─ models
|
29
|
-
│ │ │ │ └─ raw_transaction_spec.rb
|
30
|
-
│ │ └─ configuration
|
31
|
-
│ │ └─ models
|
32
|
-
│ │ └─ bank_reconciliation_config_spec.rb
|
33
|
-
└─ fixtures
|
34
|
-
└─ bank-reconciliation
|
35
|
-
└─ bank-west.csv
|
36
|
-
```
|
@@ -1,145 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Clean
|
7
|
-
# Clean transactions
|
8
|
-
class CleanTransactions
|
9
|
-
include KLog::Logging
|
10
|
-
include Appydave::Tools::Configuration::Configurable
|
11
|
-
include Appydave::Tools::Debuggable
|
12
|
-
|
13
|
-
attr_reader :transaction_folder
|
14
|
-
attr_reader :output_folder
|
15
|
-
attr_reader :transactions
|
16
|
-
|
17
|
-
# (config_file)
|
18
|
-
def initialize(transaction_folder: nil, output_folder: nil, debug: false)
|
19
|
-
@debug = debug
|
20
|
-
# needs to use config.bank_reconciliation.transaction_folder
|
21
|
-
transaction_folder ||= '/Volumes/Expansion/Sync/bank-reconciliation/original-transactions'
|
22
|
-
output_folder ||= File.join(transaction_folder, 'clean')
|
23
|
-
|
24
|
-
@transaction_folder = transaction_folder
|
25
|
-
@output_folder = output_folder
|
26
|
-
end
|
27
|
-
|
28
|
-
def clean_transactions(input_globs, output_file)
|
29
|
-
log_info("Starting transaction cleaning with input patterns: #{input_globs}")
|
30
|
-
|
31
|
-
raw_transactions = grab_raw_transactions(input_globs)
|
32
|
-
log_info("Total raw transactions collected: #{raw_transactions.size}")
|
33
|
-
|
34
|
-
transactions, duplicates_count = deduplicate_across_files(raw_transactions)
|
35
|
-
log_info("Duplicates found and removed: #{duplicates_count}")
|
36
|
-
|
37
|
-
transactions = Mapper.new.map(transactions)
|
38
|
-
log_info('Transactions mapped to chart of accounts and bank accounts')
|
39
|
-
|
40
|
-
log_kv 'Deduped consolidated transactions', duplicates_count if duplicates_count.positive?
|
41
|
-
|
42
|
-
save_to_csv(transactions, output_file)
|
43
|
-
log_kv('Transaction Output File', full_output_file(output_file))
|
44
|
-
|
45
|
-
@transactions = transactions
|
46
|
-
end
|
47
|
-
|
48
|
-
private
|
49
|
-
|
50
|
-
def grab_raw_transactions(input_globs)
|
51
|
-
original_dir = Dir.pwd
|
52
|
-
transactions = []
|
53
|
-
|
54
|
-
begin
|
55
|
-
Dir.chdir(transaction_folder)
|
56
|
-
|
57
|
-
input_globs.each do |glob|
|
58
|
-
Dir.glob(glob).each do |file|
|
59
|
-
log_kv 'Reading transactions from', file
|
60
|
-
raw_transactions = ReadTransactions.new(file).read
|
61
|
-
deduped_transactions, duplicates_count = deduplicate_within_file(raw_transactions)
|
62
|
-
|
63
|
-
if duplicates_count.positive?
|
64
|
-
log_kv 'Duplicates count within file', duplicates_count
|
65
|
-
log_kv 'File', file
|
66
|
-
end
|
67
|
-
|
68
|
-
transactions += deduped_transactions
|
69
|
-
end
|
70
|
-
end
|
71
|
-
ensure
|
72
|
-
Dir.chdir(original_dir)
|
73
|
-
end
|
74
|
-
|
75
|
-
transactions
|
76
|
-
end
|
77
|
-
|
78
|
-
def deduplicate_within_file(transactions)
|
79
|
-
unique_transactions = transactions.uniq do |transaction|
|
80
|
-
[
|
81
|
-
transaction.bsb_number,
|
82
|
-
transaction.account_number,
|
83
|
-
transaction.transaction_date,
|
84
|
-
transaction.narration,
|
85
|
-
transaction.cheque_number,
|
86
|
-
transaction.debit,
|
87
|
-
transaction.credit,
|
88
|
-
transaction.balance,
|
89
|
-
transaction.transaction_type
|
90
|
-
]
|
91
|
-
end
|
92
|
-
|
93
|
-
duplicates = transactions.size - unique_transactions.size
|
94
|
-
|
95
|
-
[unique_transactions, duplicates]
|
96
|
-
end
|
97
|
-
|
98
|
-
def deduplicate_across_files(transactions)
|
99
|
-
grouped_transactions = transactions.group_by do |transaction|
|
100
|
-
[
|
101
|
-
transaction.bsb_number,
|
102
|
-
transaction.account_number,
|
103
|
-
transaction.transaction_date,
|
104
|
-
transaction.narration,
|
105
|
-
transaction.cheque_number,
|
106
|
-
transaction.debit,
|
107
|
-
transaction.credit,
|
108
|
-
transaction.balance,
|
109
|
-
transaction.transaction_type
|
110
|
-
]
|
111
|
-
end
|
112
|
-
|
113
|
-
unique_transactions = []
|
114
|
-
duplicates_count = 0
|
115
|
-
|
116
|
-
grouped_transactions.each_value do |dupes|
|
117
|
-
unique_transaction = dupes.first
|
118
|
-
unique_transaction.source_files = dupes.flat_map(&:source_files).uniq
|
119
|
-
duplicates_count += dupes.size - 1
|
120
|
-
unique_transactions << unique_transaction
|
121
|
-
end
|
122
|
-
|
123
|
-
[unique_transactions, duplicates_count]
|
124
|
-
end
|
125
|
-
|
126
|
-
def full_output_file(output_file)
|
127
|
-
File.join(output_folder, output_file)
|
128
|
-
end
|
129
|
-
|
130
|
-
def save_to_csv(transactions, output_file)
|
131
|
-
FileUtils.mkdir_p(output_folder)
|
132
|
-
output_file = full_output_file(output_file)
|
133
|
-
|
134
|
-
CSV.open(output_file, 'w') do |csv|
|
135
|
-
csv << Appydave::Tools::BankReconciliation::Models::Transaction.csv_headers
|
136
|
-
transactions.each do |transaction|
|
137
|
-
csv << transaction.to_csv_row
|
138
|
-
end
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
143
|
-
end
|
144
|
-
end
|
145
|
-
end
|
@@ -1,136 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Clean
|
7
|
-
# Map transactions to chart of accounts and bank accounts
|
8
|
-
class Mapper
|
9
|
-
include Appydave::Tools::Configuration::Configurable
|
10
|
-
|
11
|
-
# "bank_accounts": [
|
12
|
-
# {
|
13
|
-
# "account_number": "5435 6859 0116 7736",
|
14
|
-
# "bsb": "",
|
15
|
-
# "name": "Mastercard",
|
16
|
-
# "platform": "Bankwest"
|
17
|
-
# },
|
18
|
-
# {
|
19
|
-
# "account_number": "303-092",
|
20
|
-
# "bsb": "1361644",
|
21
|
-
# "name": "atcall",
|
22
|
-
# "platform": "Bankwest"
|
23
|
-
# },
|
24
|
-
|
25
|
-
def map(transactions)
|
26
|
-
transactions.map do |original_transaction|
|
27
|
-
transaction = original_transaction.dup
|
28
|
-
|
29
|
-
transaction = map_chart_of_account(transaction)
|
30
|
-
map_bank_account(transaction)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
private
|
35
|
-
|
36
|
-
def map_chart_of_account(transaction)
|
37
|
-
equality_match(transaction) ||
|
38
|
-
trigram_match(transaction, 0.9, '90%') ||
|
39
|
-
trigram_match(transaction, 0.8, '80%') ||
|
40
|
-
trigram_match(transaction, 0.7, '70%') ||
|
41
|
-
trigram_match(transaction, 0.6, '60%') ||
|
42
|
-
trigram_match(transaction, 0.5, '50%') ||
|
43
|
-
start_with_match(transaction) ||
|
44
|
-
includes(transaction)
|
45
|
-
transaction
|
46
|
-
end
|
47
|
-
|
48
|
-
def map_bank_account(transaction)
|
49
|
-
bank_account = config.bank_reconciliation.get_bank_account(transaction.account_number, transaction.bsb_number)
|
50
|
-
|
51
|
-
if bank_account
|
52
|
-
transaction.account_name = bank_account.name
|
53
|
-
transaction.platform = bank_account.platform
|
54
|
-
end
|
55
|
-
|
56
|
-
transaction
|
57
|
-
end
|
58
|
-
|
59
|
-
def equality_match(transaction)
|
60
|
-
coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
|
61
|
-
chart_of_account.narration.to_s.delete(' ').downcase == transaction.narration.delete(' ').downcase
|
62
|
-
end
|
63
|
-
|
64
|
-
return nil unless coa
|
65
|
-
|
66
|
-
transaction.coa_match_type = 'equality'
|
67
|
-
transaction.coa_code = coa.code
|
68
|
-
transaction
|
69
|
-
end
|
70
|
-
|
71
|
-
def start_with_match(transaction)
|
72
|
-
coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
|
73
|
-
transaction.narration.to_s.delete(' ').downcase.start_with?(chart_of_account.narration.to_s.downcase)
|
74
|
-
end
|
75
|
-
|
76
|
-
return nil unless coa
|
77
|
-
|
78
|
-
transaction.coa_match_type = 'starts_with'
|
79
|
-
transaction.coa_code = coa.code
|
80
|
-
transaction
|
81
|
-
end
|
82
|
-
|
83
|
-
def includes(transaction)
|
84
|
-
coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
|
85
|
-
transaction.narration.to_s.delete(' ').downcase.include?(chart_of_account.narration.delete(' ').to_s.downcase)
|
86
|
-
end
|
87
|
-
|
88
|
-
return nil unless coa
|
89
|
-
|
90
|
-
transaction.coa_match_type = 'includes'
|
91
|
-
transaction.coa_code = coa.code
|
92
|
-
transaction
|
93
|
-
end
|
94
|
-
|
95
|
-
def trigram_match(transaction, score_threshold, match_type)
|
96
|
-
scored_transactions = config.bank_reconciliation.chart_of_accounts.map do |coa|
|
97
|
-
{
|
98
|
-
coa: coa,
|
99
|
-
score: compare(coa.narration, transaction.narration)
|
100
|
-
}
|
101
|
-
end
|
102
|
-
|
103
|
-
scored_transactions.sort_by! { |t| t[:score] }.reverse!
|
104
|
-
|
105
|
-
best = scored_transactions.first
|
106
|
-
|
107
|
-
return nil unless best
|
108
|
-
return nil if best[:score] < score_threshold
|
109
|
-
|
110
|
-
coa = best[:coa]
|
111
|
-
|
112
|
-
transaction.coa_match_type = match_type
|
113
|
-
transaction.coa_code = coa.code
|
114
|
-
transaction
|
115
|
-
end
|
116
|
-
|
117
|
-
def compare(text1, text2)
|
118
|
-
text1_trigs = trigramify(text1)
|
119
|
-
text2_trigs = trigramify(text2)
|
120
|
-
|
121
|
-
all_cnt = (text1_trigs | text2_trigs).size
|
122
|
-
same_cnt = (text1_trigs & text2_trigs).size
|
123
|
-
|
124
|
-
same_cnt.to_f / all_cnt
|
125
|
-
end
|
126
|
-
|
127
|
-
def trigramify(text)
|
128
|
-
trigs = []
|
129
|
-
text.chars.each_cons(3) { |v| trigs << v.join }
|
130
|
-
trigs
|
131
|
-
end
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
@@ -1,90 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Clean
|
7
|
-
# Read transactions from a CSV file
|
8
|
-
class ReadTransactions
|
9
|
-
attr_reader :platform
|
10
|
-
attr_reader :transactions
|
11
|
-
|
12
|
-
def initialize(file)
|
13
|
-
@file = file
|
14
|
-
end
|
15
|
-
|
16
|
-
def read
|
17
|
-
csv_lines = File.read(@file).lines
|
18
|
-
|
19
|
-
@platform = detect_platform(csv_lines)
|
20
|
-
|
21
|
-
case platform
|
22
|
-
when :bankwest
|
23
|
-
read_bankwest(csv_lines)
|
24
|
-
when :bankwest2
|
25
|
-
read_bankwest2(csv_lines)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
private
|
30
|
-
|
31
|
-
def read_bankwest(csv_lines)
|
32
|
-
@transactions = []
|
33
|
-
|
34
|
-
# Skip the header line and parse each subsequent line
|
35
|
-
CSV.parse(csv_lines.join, headers: true).each do |row|
|
36
|
-
transaction = Models::Transaction.new(
|
37
|
-
bsb_number: row['BSB Number'],
|
38
|
-
account_number: row['Account Number'],
|
39
|
-
transaction_date: row['Transaction Date'],
|
40
|
-
narration: row['Narration'],
|
41
|
-
cheque_number: row['Cheque Number'],
|
42
|
-
debit: row['Debit'],
|
43
|
-
credit: row['Credit'],
|
44
|
-
balance: row['Balance'],
|
45
|
-
transaction_type: row['Transaction Type']
|
46
|
-
)
|
47
|
-
transaction.add_source_file(@file)
|
48
|
-
@transactions << transaction
|
49
|
-
end
|
50
|
-
|
51
|
-
@transactions
|
52
|
-
end
|
53
|
-
|
54
|
-
def read_bankwest2(csv_lines)
|
55
|
-
@transactions = []
|
56
|
-
|
57
|
-
# Skip the header line and parse each subsequent line
|
58
|
-
CSV.parse(csv_lines.join, headers: true).each do |row|
|
59
|
-
transaction = Models::Transaction.new(
|
60
|
-
bsb_number: row['BSB / Account Number'].split(' - ').first,
|
61
|
-
account_number: row['BSB / Account Number'].split(' - ').last,
|
62
|
-
transaction_date: row['Transaction Date'],
|
63
|
-
narration: row['Narration'],
|
64
|
-
cheque_number: row['Cheque Number'],
|
65
|
-
debit: row['Debit'],
|
66
|
-
credit: row['Credit'],
|
67
|
-
balance: row['Balance'],
|
68
|
-
transaction_type: row['Transaction Type']
|
69
|
-
)
|
70
|
-
transaction.add_source_file(@file)
|
71
|
-
@transactions << transaction
|
72
|
-
end
|
73
|
-
|
74
|
-
@transactions
|
75
|
-
end
|
76
|
-
|
77
|
-
# For bankwest the first row is the CSV will look like:
|
78
|
-
# BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type
|
79
|
-
def detect_platform(csv_lines)
|
80
|
-
return :bankwest if csv_lines.first.start_with?('BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
|
81
|
-
return :bankwest2 if csv_lines.first.start_with?('Account Name,BSB / Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
|
82
|
-
|
83
|
-
puts "Unknown platform detected. CSV columns are: #{csv_lines.first.strip}"
|
84
|
-
raise Appydave::Tools::Error, 'Unknown platform'
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
@@ -1,101 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module BankReconciliation
|
6
|
-
module Models
|
7
|
-
# Unified transaction model for raw and reconciled data
|
8
|
-
class Transaction
|
9
|
-
attr_accessor :bsb_number,
|
10
|
-
:account_number,
|
11
|
-
:transaction_date,
|
12
|
-
:narration,
|
13
|
-
:cheque_number,
|
14
|
-
:debit,
|
15
|
-
:credit,
|
16
|
-
:balance,
|
17
|
-
:transaction_type,
|
18
|
-
:platform,
|
19
|
-
:coa_code,
|
20
|
-
:coa_match_type,
|
21
|
-
:account_name,
|
22
|
-
:source_files
|
23
|
-
|
24
|
-
def initialize(bsb_number: nil,
|
25
|
-
account_number: nil,
|
26
|
-
transaction_date: nil,
|
27
|
-
narration: nil,
|
28
|
-
cheque_number: nil,
|
29
|
-
debit: nil,
|
30
|
-
credit: nil,
|
31
|
-
balance: nil,
|
32
|
-
transaction_type: nil,
|
33
|
-
platform: nil,
|
34
|
-
coa_code: nil,
|
35
|
-
coa_match_type: nil,
|
36
|
-
account_name: nil)
|
37
|
-
@bsb_number = bsb_number&.strip
|
38
|
-
@account_number = account_number&.strip
|
39
|
-
@transaction_date = transaction_date&.strip
|
40
|
-
@transaction_date = Date.strptime(@transaction_date, '%d/%m/%Y')
|
41
|
-
@narration = narration&.gsub(/\s{2,}/, ' ')&.strip
|
42
|
-
@cheque_number = cheque_number&.strip
|
43
|
-
@debit = debit&.strip
|
44
|
-
@credit = credit&.strip
|
45
|
-
@balance = balance&.strip
|
46
|
-
@transaction_type = transaction_type&.strip
|
47
|
-
@platform = platform
|
48
|
-
@coa_code = coa_code
|
49
|
-
@coa_match_type = coa_match_type
|
50
|
-
@account_name = account_name
|
51
|
-
@source_files = []
|
52
|
-
end
|
53
|
-
|
54
|
-
def add_source_file(source_file)
|
55
|
-
@source_files << source_file.strip unless @source_files.include?(source_file.strip)
|
56
|
-
end
|
57
|
-
|
58
|
-
# cheque_number
|
59
|
-
|
60
|
-
def self.csv_headers
|
61
|
-
%i[
|
62
|
-
platform
|
63
|
-
account_name
|
64
|
-
bsb_number
|
65
|
-
account_number
|
66
|
-
transaction_date
|
67
|
-
narration
|
68
|
-
debit
|
69
|
-
credit
|
70
|
-
balance
|
71
|
-
transaction_type
|
72
|
-
coa_code
|
73
|
-
coa_match_type
|
74
|
-
source_files
|
75
|
-
]
|
76
|
-
end
|
77
|
-
|
78
|
-
# @cheque_number,
|
79
|
-
|
80
|
-
def to_csv_row
|
81
|
-
[
|
82
|
-
@platform,
|
83
|
-
@account_name,
|
84
|
-
@bsb_number,
|
85
|
-
@account_number,
|
86
|
-
@transaction_date,
|
87
|
-
@narration,
|
88
|
-
@debit,
|
89
|
-
@credit,
|
90
|
-
@balance,
|
91
|
-
@transaction_type,
|
92
|
-
@coa_code,
|
93
|
-
@coa_match_type,
|
94
|
-
@source_files
|
95
|
-
]
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
@@ -1,97 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Appydave
|
4
|
-
module Tools
|
5
|
-
module Configuration
|
6
|
-
module Models
|
7
|
-
# Bank reconciliation configuration
|
8
|
-
class BankReconciliationConfig < ConfigBase
|
9
|
-
# def
|
10
|
-
# Retrieve all bank accounts
|
11
|
-
def bank_accounts
|
12
|
-
data['bank_accounts'].map do |account|
|
13
|
-
BankAccount.new(account)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
def chart_of_accounts
|
18
|
-
data['chart_of_accounts'].map do |entry|
|
19
|
-
ChartOfAccount.new(entry)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
def get_bank_account(account_number, bsb = nil)
|
24
|
-
account_data = data['bank_accounts'].find do |account|
|
25
|
-
account['account_number'] == account_number && (account['bsb'].nil? || account['bsb'] == bsb)
|
26
|
-
end
|
27
|
-
|
28
|
-
BankAccount.new(account_data) if account_data
|
29
|
-
end
|
30
|
-
|
31
|
-
# Retrieve a chart of account entry by code
|
32
|
-
def get_chart_of_account(code)
|
33
|
-
entry_data = data['chart_of_accounts'].find { |entry| entry['code'] == code }
|
34
|
-
ChartOfAccount.new(entry_data) if entry_data
|
35
|
-
end
|
36
|
-
|
37
|
-
def print
|
38
|
-
log.subheading 'Bank Reconciliation - Accounts'
|
39
|
-
|
40
|
-
tp bank_accounts, :account_number, :bsb, :name, :bank
|
41
|
-
|
42
|
-
log.subheading 'Bank Reconciliation - Chart of Accounts'
|
43
|
-
|
44
|
-
tp chart_of_accounts, :code, :narration
|
45
|
-
end
|
46
|
-
|
47
|
-
private
|
48
|
-
|
49
|
-
def default_data
|
50
|
-
{
|
51
|
-
'bank_accounts' => [],
|
52
|
-
'chart_of_accounts' => []
|
53
|
-
}
|
54
|
-
end
|
55
|
-
|
56
|
-
# Inner class to represent a bank account
|
57
|
-
class BankAccount
|
58
|
-
attr_accessor :account_number, :bsb, :name, :platform
|
59
|
-
|
60
|
-
def initialize(data)
|
61
|
-
@account_number = data['account_number']
|
62
|
-
@bsb = data['bsb']
|
63
|
-
@name = data['name']
|
64
|
-
@platform = data['platform']
|
65
|
-
end
|
66
|
-
|
67
|
-
def to_h
|
68
|
-
{
|
69
|
-
'account_number' => @account_number,
|
70
|
-
'bsb' => @bsb,
|
71
|
-
'name' => @name,
|
72
|
-
'platform' => @platform
|
73
|
-
}
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
# Inner class to represent a chart of account entry
|
78
|
-
class ChartOfAccount
|
79
|
-
attr_accessor :code, :narration
|
80
|
-
|
81
|
-
def initialize(data)
|
82
|
-
@code = data['code']
|
83
|
-
@narration = data['narration']
|
84
|
-
end
|
85
|
-
|
86
|
-
def to_h
|
87
|
-
{
|
88
|
-
'code' => @code,
|
89
|
-
'narration' => @narration
|
90
|
-
}
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|