cl-magic 0.4.0 → 1.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '08b4655abcd92a0d5e3a779fdc7e0e09231c7cb15d073e06878c085227ef3f9d'
4
- data.tar.gz: 63999d535d6ad18d22b8cff1e259b9b054bbf7faf75939e54c9819885ca24feb
3
+ metadata.gz: bae4cae1197514890ab80299d8f0379f89661e2b8d731ac714e0b99cdcd7f4ba
4
+ data.tar.gz: 47588e897f238d68a9edbbb3bd13ca418018c85aade5006bd9ea7c1f742bf192
5
5
  SHA512:
6
- metadata.gz: 4157d207e824e2d0a309790187873d3b6b02c41bd892fc3436e359cc1f4639044576129d1a8d36ac9a0faee8afa6dc2366db506f93401bf4165e93766435f3d7
7
- data.tar.gz: 0bce3b21105e81dd1bc1d5656e941ceec30814bb01e19c16a7a5ee57eba98403093d7359eb9e113bd60b0850a87764b1241a08febe1881dcde74c303fb9d35be
6
+ metadata.gz: 1446ea456116138116613b93d732ee02dfead51c66df386be273b36dc660bc762892d9bc6b110cadab620d4b6023922ce4a5e79c3261230ddfffd8c685f577db
7
+ data.tar.gz: bc3d680c504cf92f88ed5da9743068256c9245234d86e656c272b168e69ea611d45a8ed0d976ea6033738bd4eb7e69fd3cbd38cd10238e3030632450db53699f
data/Gemfile.lock CHANGED
@@ -1,22 +1,27 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- cl-magic (0.4.0)
4
+ cl-magic (1.2.1)
5
5
  activesupport
6
+ baran
7
+ concurrent-ruby
6
8
  optparse-subcommand
7
9
  pastel
8
10
  tty-command
9
11
  tty-logger
12
+ tty-progressbar
10
13
  tty-prompt
14
+ tty-spinner
11
15
 
12
16
  GEM
13
17
  remote: https://rubygems.org/
14
18
  specs:
15
- activesupport (7.0.6)
19
+ activesupport (7.0.7)
16
20
  concurrent-ruby (~> 1.0, >= 1.0.2)
17
21
  i18n (>= 1.6, < 2)
18
22
  minitest (>= 5.1)
19
23
  tzinfo (~> 2.0)
24
+ baran (0.1.7)
20
25
  byebug (11.1.3)
21
26
  concurrent-ruby (1.2.2)
22
27
  i18n (1.14.1)
@@ -26,12 +31,18 @@ GEM
26
31
  pastel (0.8.0)
27
32
  tty-color (~> 0.5)
28
33
  rake (13.0.6)
34
+ strings-ansi (0.2.0)
29
35
  tty-color (0.6.0)
30
36
  tty-command (0.10.1)
31
37
  pastel (~> 0.8)
32
38
  tty-cursor (0.7.1)
33
39
  tty-logger (0.6.0)
34
40
  pastel (~> 0.8)
41
+ tty-progressbar (0.18.2)
42
+ strings-ansi (~> 0.2)
43
+ tty-cursor (~> 0.7)
44
+ tty-screen (~> 0.8)
45
+ unicode-display_width (>= 1.6, < 3.0)
35
46
  tty-prompt (0.23.1)
36
47
  pastel (~> 0.8)
37
48
  tty-reader (~> 0.8)
@@ -40,8 +51,11 @@ GEM
40
51
  tty-screen (~> 0.8)
41
52
  wisper (~> 2.0)
42
53
  tty-screen (0.8.1)
54
+ tty-spinner (0.9.3)
55
+ tty-cursor (~> 0.7)
43
56
  tzinfo (2.0.6)
44
57
  concurrent-ruby (~> 1.0)
58
+ unicode-display_width (2.4.2)
45
59
  wisper (2.0.1)
46
60
 
47
61
  PLATFORMS
data/cl-magic.gemspec CHANGED
@@ -33,8 +33,13 @@ Gem::Specification.new do |spec|
33
33
  spec.add_dependency "tty-logger"
34
34
  spec.add_dependency "tty-command"
35
35
  spec.add_dependency "tty-prompt"
36
+ spec.add_dependency "tty-spinner"
37
+ spec.add_dependency "tty-progressbar"
38
+ spec.add_dependency "concurrent-ruby"
36
39
  spec.add_dependency "pastel"
37
40
  spec.add_dependency "activesupport"
41
+ spec.add_dependency "baran"
42
+
38
43
  spec.add_development_dependency "byebug"
39
44
 
40
45
  end
data/lib/cl/magic/cl CHANGED
@@ -3,7 +3,6 @@ require 'optparse'
3
3
  require 'tty-command'
4
4
  require 'cl/magic/common/logging.rb'
5
5
 
6
-
7
6
  @logger = get_logger()
8
7
 
9
8
  #
@@ -76,26 +75,34 @@ def print_commands(commands, prefix=' ')
76
75
  end
77
76
  end
78
77
 
79
- def execute_subcommand(subcommand, command_tree)
80
- commands = command_tree.fetch(subcommand, [])
78
+ def execute_subcommand(cmd, command_tree)
79
+ # global command?
80
+ global_commands = command_tree['']
81
+ global_cmd = global_commands.select {|c| c[:cmd]==cmd}.first
82
+
83
+ # subcommand?
84
+ sub_commands = command_tree.fetch(cmd, [])
85
+ next_arg = ARGV.first
86
+ sub_cmd = sub_commands.select {|c| c[:cmd]==next_arg}.first
81
87
 
82
- case commands.length
83
- when 0
84
- commands = command_tree[''] # global command?
88
+ # found sub command?
89
+ if sub_cmd
90
+ ARGV.shift
91
+ cmd = sub_cmd
85
92
  else
86
- commands = command_tree.fetch(subcommand, [])
87
- sub_subcommand = ARGV.shift if commands.any? # next part of command
88
- if sub_subcommand.nil?
93
+ # found global command?
94
+ if global_cmd
95
+ cmd = global_cmd
96
+ else
97
+ # else print help
89
98
  puts ""
90
- puts "#{subcommand}"
91
- print_commands(commands)
99
+ puts "#{cmd}"
100
+ print_commands(sub_commands)
92
101
  puts ""
93
102
  exit(0)
94
- else
95
- subcommand = sub_subcommand
96
103
  end
97
104
  end
98
- cmd = commands.select {|c| c[:cmd]==subcommand}.first
105
+
99
106
  final = "bundle exec ./#{cmd[:filename]} #{ARGV.join(' ')}"
100
107
  exec(final)
101
108
  end
@@ -0,0 +1,117 @@
1
+ #!/usr/bin/env ruby
2
+ # Call open ai chat api
3
+ require 'optparse'
4
+ require 'optparse/subcommand'
5
+ require 'tty-command'
6
+ require 'tty-prompt'
7
+
8
+ require 'cl/magic/common/common_options.rb'
9
+ require 'cl/magic/common/logging.rb'
10
+ require 'cl/magic/common/ai_prompt.rb'
11
+
12
+ @logger = get_logger()
13
+ @cl_cmd_name = File.basename(__FILE__).split('-').join(' ')
14
+
15
+ def do_prompt(options, ai, data)
16
+ @logger.info("#{data.length} chars")
17
+ data = ai.prompt(data, options[:prompt], options[:split_as_markdown], options[:split_separator])
18
+ return data if data.count == 1 or not options[:recurse]
19
+
20
+ # recurse
21
+ options[:split_as_markdown] = false # disable on second pass
22
+ return do_prompt(options, ai, data.join("\n"))
23
+ end
24
+
25
+ #
26
+ # Features
27
+ #
28
+
29
+ def do_work(options, data)
30
+ ai = AIPrompt.new(@logger, @working_dir, options[:max_chunk_size], options[:temperature])
31
+ ai.clear_cache()
32
+
33
+ @logger.info "max_chunk_size: #{options[:max_chunk_size]} | temperature: #{options[:temperature]}"
34
+ @logger.info "prompt: #{options[:prompt]}"
35
+
36
+ data = do_prompt(options, ai, data)
37
+ data.each do |json|
38
+ @logger.puts ""
39
+ puts json
40
+ end
41
+ end
42
+
43
+ #
44
+ # Options
45
+ #
46
+
47
+ options = {
48
+ max_chunk_size: 10000,
49
+ temperature: 1.0
50
+ }
51
+ global_banner = <<DOC
52
+
53
+ Chat API - https://platform.openai.com/docs/api-reference/chat
54
+
55
+ Usage: #{@cl_cmd_name} [options] -- prompt
56
+
57
+ DOC
58
+
59
+ global = OptionParser.new do |g|
60
+ g.banner = global_banner
61
+ add_help_and_verbose(g)
62
+
63
+ g.on("-f", "--filepath FILEPATH", "relative path to file containing prompt data") do |v|
64
+ options[:data_filepath] = v
65
+ end
66
+
67
+ g.on("-m", "--max-chunk SIZE", "(default: 10,000) lower this if you hit token limit with LLM") do |v|
68
+ options[:max_chunk_size] = v.to_i
69
+ end
70
+
71
+ g.on("-t", "--temperature NUMBER", "(default: 1.0) lower for more deterministic results; whatever that means?") do |v|
72
+ options[:temperature] = v.to_f
73
+ end
74
+
75
+ g.on("--split-separator VALUE", "(default: '\\n\\n') string to split text by") do |v|
76
+ options[:split_separator] = v
77
+ end
78
+
79
+ g.on("--split-md", "optional, split by markdown") do |v|
80
+ options[:split_as_markdown] = v
81
+ end
82
+
83
+ g.on("--recurse", "optional, reduce down to 1 chunk") do |v|
84
+ options[:recurse] = v
85
+ end
86
+ end
87
+
88
+ #
89
+ # Run
90
+ #
91
+
92
+ @working_dir = ENV['CL_WORKING_DIR'] # passed through cl-magic to here
93
+ global.parse!(ARGV)
94
+ options[:prompt] = ARGV.join(' ')
95
+
96
+ # keys
97
+ ["OPENAPI_KEY", "OPENAPI_URL"].each do |env_key|
98
+ unless ENV[env_key]
99
+ @logger.error "missing environment variable: #{env_key}"
100
+ exit 1
101
+ end
102
+ end
103
+
104
+ if options[:data_filepath].nil?
105
+ puts global.parse! %w[--help]
106
+ exit 1
107
+ end
108
+
109
+ ask_and_store_option(options, :prompt, "prompt?")
110
+
111
+ # chat
112
+ filepath = File.join(@working_dir, options[:data_filepath])
113
+ data = File.readlines(filepath).join("\n")
114
+ do_work(options, data)
115
+
116
+ # history
117
+ write_history("""#{@cl_cmd_name} --filepath #{options[:data_filepath]} -- #{options[:prompt]}""")
@@ -0,0 +1,116 @@
1
+ #!/usr/bin/env ruby
2
+ # Prompt for similar content
3
+ require 'optparse'
4
+ require 'optparse/subcommand'
5
+ require 'tty-command'
6
+ require 'tty-prompt'
7
+
8
+ require 'cl/magic/common/common_options.rb'
9
+ require 'cl/magic/common/logging.rb'
10
+ require 'cl/magic/common/jira.rb'
11
+ require 'cl/magic/common/ai_prompt.rb'
12
+ require 'cl/magic/common/milvus.rb'
13
+ require 'cl/magic/common/elastic.rb'
14
+
15
+ @logger = get_logger()
16
+
17
+ #
18
+ # Features
19
+ #
20
+
21
+ def do_work(options)
22
+
23
+ # embedding from prompt
24
+ @logger.wait "convert prompt to embedding"
25
+ ai = AIPrompt.new(@logger, @working_dir)
26
+ embedding = ai.gen_embeddings(options[:prompt])
27
+
28
+ # search
29
+ @logger.wait "search vector db"
30
+ milvus = Milvus.new(options[:milvus_host], options[:milvus_port])
31
+ collection_name = "jira_markdown"
32
+ response = milvus.search(collection_name, embedding)
33
+
34
+ # doc keys
35
+ data = JSON.parse(response)["data"]
36
+ data = data.select{|h| h["distance"] > 0.6} # reduce
37
+ doc_keys = data.map {|h| h["doc_key"]}
38
+
39
+ # elastic content
40
+ @logger.wait "fetch documents from elastic"
41
+ elastic = Elastic.new(options[:elastic_url])
42
+ response = elastic.query_by_id(doc_keys)
43
+ docs = JSON.parse(response)
44
+ puts docs["hits"]["hits"].map { |h| h["_source"]["markdown"] }.join("\n\n")
45
+ end
46
+
47
+ #
48
+ # Options
49
+ #
50
+
51
+ options = {}
52
+
53
+ global_banner = <<DOC
54
+
55
+ Prompt for similar content
56
+
57
+ Usage: cl ai query [options]
58
+
59
+ DOC
60
+
61
+ global = OptionParser.new do |g|
62
+ g.banner = global_banner
63
+ add_help_and_verbose(g)
64
+
65
+ g.on("--milvus-host HOST", "host for milvus vector db") do |v|
66
+ options[:milvus_host] = v
67
+ end
68
+
69
+ g.on("--milvus-port PORT", "port for milvus vector db") do |v|
70
+ options[:milvus_port] = v
71
+ end
72
+
73
+ g.on("--elastic URL", "url to elastic search server") do |v|
74
+ options[:elastic_url] = v
75
+ end
76
+
77
+ end
78
+
79
+ #
80
+ # Run
81
+ #
82
+
83
+ @working_dir = ENV['CL_WORKING_DIR'] # passed through cl-magic to here
84
+ global.parse!(ARGV)
85
+ options[:prompt] = ARGV.join(' ')
86
+
87
+ # keys
88
+ ["OPENAPI_KEY", "OPENAPI_URL"].each do |env_key|
89
+ unless ENV[env_key]
90
+ @logger.error "missing environment variable: #{env_key}"
91
+ exit 1
92
+ end
93
+ end
94
+
95
+ # required options
96
+ [
97
+ :milvus_host,
98
+ :milvus_port,
99
+ :elastic_url,
100
+ ].each do |key|
101
+ if options[key].nil?
102
+ puts global.parse! %w[--help]
103
+ exit 1
104
+ end
105
+ end
106
+
107
+ ask_and_store_option(options, :prompt, "prompt?")
108
+
109
+ write_history("""cl ai query \\
110
+ --elastic #{options[:elastic_url]} \\
111
+ --milvus-host #{options[:milvus_host]} \\
112
+ --milvus-port #{options[:milvus_port]} \\
113
+ -- #{options[:prompt]}
114
+ """)
115
+
116
+ do_work(options)
@@ -0,0 +1,158 @@
1
+ #!/usr/bin/env ruby
2
+ # Store jira json for AI
3
+ require 'optparse'
4
+ require 'optparse/subcommand'
5
+ require 'tty-command'
6
+ require 'tty-prompt'
7
+
8
+ require 'cl/magic/common/common_options.rb'
9
+ require 'cl/magic/common/logging.rb'
10
+ require 'cl/magic/common/jira.rb'
11
+ require 'cl/magic/common/ai_prompt.rb'
12
+ require 'cl/magic/common/milvus.rb'
13
+ require 'cl/magic/common/elastic.rb'
14
+
15
+ @logger = get_logger()
16
+
17
+ def store_elastic(elastic, elastic_index, issues)
18
+ issues.each do |i|
19
+ issue_md, comments = Jira.jira_to_markdown(i)
20
+
21
+ # issue
22
+ url = "#{elastic_index}/_doc/issue_#{i["id"]}"
23
+ @logger.puts elastic.post(url, "POST", {
24
+ "markdown": issue_md
25
+ })
26
+
27
+ comments.each do |comment|
28
+
29
+ # comment
30
+ id, comment_md = comment
31
+ url = "#{elastic_index}/_doc/comment_#{id}"
32
+ @logger.puts elastic.post(url, "POST", {
33
+ "markdown": comment_md
34
+ })
35
+
36
+ end
37
+ end
38
+ end
39
+
40
+ def store_vector(ai, milvus, issues)
41
+ collection_name = "jira_markdown"
42
+ @logger.puts milvus.create_collection(collection_name)
43
+
44
+ issues.each do |i|
45
+ issue_md, comments = Jira.jira_to_markdown(i)
46
+
47
+ # issue
48
+ doc_key = "issue_#{i["id"]}"
49
+ embedding = ai.gen_embeddings(issue_md)
50
+ @logger.puts milvus.post_to_collection(collection_name, doc_key, embedding)
51
+
52
+ comments.each do |comment|
53
+ comment_id, comment_md = comment
54
+
55
+ # comment
56
+ doc_key = "comment_#{comment_id}"
57
+ embedding = ai.gen_embeddings(issue_md)
58
+ @logger.puts milvus.post_to_collection(collection_name, doc_key, embedding)
59
+ end
60
+ end
61
+ end
62
+
63
+ #
64
+ # Features
65
+ #
66
+
67
+ def do_work(options)
68
+
69
+ filepath = File.join(@working_dir, options[:data_filepath])
70
+ issues = JSON.parse(File.read(filepath))
71
+
72
+ # elastic
73
+ elastic = Elastic.new(options[:elastic_url])
74
+ index = options[:elastic_index]
75
+ @logger.puts elastic.create_index(index, {
76
+ "mappings": {
77
+ "properties": {
78
+ "markdown": {
79
+ "type": "text"
80
+ }
81
+ }
82
+ }
83
+ })
84
+ store_elastic(elastic, index, issues)
85
+
86
+ # vector
87
+ ai = AIPrompt.new(@logger, @working_dir)
88
+ milvus = Milvus.new(options[:milvus_host], options[:milvus_port])
89
+ store_vector(ai, milvus, issues)
90
+ end
91
+
92
+ #
93
+ # Options
94
+ #
95
+
96
+ options = {
97
+ elastic_index: "jira_markdown"
98
+ }
99
+
100
+ global_banner = <<DOC
101
+
102
+ Store a jira json for the AI
103
+
104
+ Usage: cl ai store-jira [options]
105
+
106
+ DOC
107
+
108
+ global = OptionParser.new do |g|
109
+ g.banner = global_banner
110
+ add_help_and_verbose(g)
111
+
112
+ g.on("--milvus-host HOST", "host for milvus vector db") do |v|
113
+ options[:milvus_host] = v
114
+ end
115
+
116
+ g.on("--milvus-port PORT", "port for milvus vector db") do |v|
117
+ options[:milvus_port] = v
118
+ end
119
+
120
+ g.on("--elastic URL", "url to elastic search server") do |v|
121
+ options[:elastic_url] = v
122
+ end
123
+
124
+ g.on("-f", "--filepath FILEPATH", "relative path to jira json") do |v|
125
+ options[:data_filepath] = v
126
+ end
127
+
128
+ end
129
+
130
+ #
131
+ # Run
132
+ #
133
+
134
+ @working_dir = ENV['CL_WORKING_DIR'] # passed through cl-magic to here
135
+ global.parse!(ARGV)
136
+
137
+ # required options
138
+ [
139
+ :data_filepath,
140
+ :milvus_host,
141
+ :milvus_port,
142
+ :elastic_url,
143
+ :data_filepath
144
+ ].each do |key|
145
+ if options[key].nil?
146
+ puts global.parse! %w[--help]
147
+ exit 1
148
+ end
149
+ end
150
+
151
+ write_history("""cl ai store-jira \\
152
+ --elastic #{options[:elastic_url]} \\
153
+ --filepath #{options[:data_filepath]} \\
154
+ --milvus-host #{options[:milvus_host]} \\
155
+ --milvus-port #{options[:milvus_port]}
156
+ """)
157
+
158
+ do_work(options)