skill 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +7 -0
- data/LICENSE +21 -0
- data/README.md +280 -0
- data/exe/skill +7 -0
- data/lib/skill/agents.rb +148 -0
- data/lib/skill/archive.rb +143 -0
- data/lib/skill/boundary.rb +42 -0
- data/lib/skill/cli.rb +90 -0
- data/lib/skill/commands/add.rb +139 -0
- data/lib/skill/commands/agents.rb +19 -0
- data/lib/skill/commands/command.rb +84 -0
- data/lib/skill/commands/find.rb +51 -0
- data/lib/skill/commands/list.rb +73 -0
- data/lib/skill/commands/remove.rb +59 -0
- data/lib/skill/commands/selection.rb +87 -0
- data/lib/skill/commands/update.rb +175 -0
- data/lib/skill/commands.rb +15 -0
- data/lib/skill/directory.rb +59 -0
- data/lib/skill/discovery.rb +166 -0
- data/lib/skill/fetcher.rb +62 -0
- data/lib/skill/github.rb +158 -0
- data/lib/skill/http.rb +76 -0
- data/lib/skill/installer.rb +92 -0
- data/lib/skill/limits.rb +40 -0
- data/lib/skill/lockfile.rb +112 -0
- data/lib/skill/manifest.rb +103 -0
- data/lib/skill/session.rb +88 -0
- data/lib/skill/source.rb +137 -0
- data/lib/skill/style.rb +28 -0
- data/lib/skill/targets.rb +173 -0
- data/lib/skill/tree.rb +90 -0
- data/lib/skill/version.rb +5 -0
- data/lib/skill/workspace.rb +161 -0
- data/lib/skill.rb +60 -0
- metadata +84 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Skill
|
|
4
|
+
module Commands
|
|
5
|
+
# skill update [names]: fetches each skill's source again at the ref it
|
|
6
|
+
# was installed from. A commit never moves. A tag is fetched again and a
|
|
7
|
+
# tag that moved is called out. A branch moves forward. Nothing changes
|
|
8
|
+
# before the changes are shown and confirmed, with a prompt on a
|
|
9
|
+
# terminal or +--yes+ elsewhere.
|
|
10
|
+
class Update < Command
|
|
11
|
+
# One skill's update: the lockfile entry it will have, the copy to
|
|
12
|
+
# install, and the files that change.
|
|
13
|
+
Change = Struct.new(:name, :entry, :copy, :diff, :from, :to, :source, keyword_init: true)
|
|
14
|
+
|
|
15
|
+
NO_DIFF = Tree::Diff.new(added: [].freeze, changed: [].freeze, removed: [].freeze).freeze
|
|
16
|
+
|
|
17
|
+
def run
|
|
18
|
+
names = parse('update [names] [options]') { |opts| define(opts) }
|
|
19
|
+
lock = workspace.lock
|
|
20
|
+
names = lock.names if names.empty?
|
|
21
|
+
unknown = names - lock.names
|
|
22
|
+
raise NotFoundError, "Not installed: #{unknown.join(', ')}. Run `skill list`" if unknown.any?
|
|
23
|
+
return say('No skills installed.') if names.empty?
|
|
24
|
+
|
|
25
|
+
changes = names.group_by { |name| lock[name].values_at('source', 'ref', 'ref_type') }.flat_map do |key, group|
|
|
26
|
+
plan(*key, group)
|
|
27
|
+
end
|
|
28
|
+
apply(changes)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def define(opts)
|
|
34
|
+
opts.on('-n', '--dry-run', 'Show what would change, without changing anything') { options[:dry_run] = true }
|
|
35
|
+
opts.on('-y', '--yes', 'Apply the changes without asking') { options[:yes] = true }
|
|
36
|
+
opts.on('-f', '--force', 'Replace skills that were edited since they were installed') do
|
|
37
|
+
options[:force] = true
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Returns the Changes for the skills +names+, which share a source and
|
|
42
|
+
# a ref, fetching the source only when its ref moved.
|
|
43
|
+
def plan(source_label, ref, ref_type, names)
|
|
44
|
+
lock = workspace.lock
|
|
45
|
+
return pinned(names) if ref_type == 'commit'
|
|
46
|
+
|
|
47
|
+
source = Source.parse(source_label, root: workspace.root, ref: ref)
|
|
48
|
+
resolution = session.fetcher.resolve(source) if source.github?
|
|
49
|
+
if resolution && names.all? { |name| lock[name]['commit'] == resolution.commit }
|
|
50
|
+
at = style.dim("(#{resolution.label} at #{resolution.commit[0, 12]})")
|
|
51
|
+
names.each { |name| say done("#{style.bold(name)} up to date #{at}") }
|
|
52
|
+
return []
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
checkout = session.fetcher.checkout(source, resolution: resolution)
|
|
56
|
+
warn_moved_tag(names, checkout) if ref_type == 'tag'
|
|
57
|
+
names.filter_map { |name| change(checkout, name) }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def pinned(names)
|
|
61
|
+
names.each do |name|
|
|
62
|
+
commit = workspace.lock[name]['commit'][0, 12]
|
|
63
|
+
say "#{style.dim('·')} #{style.bold(name)} pinned #{style.dim("to #{commit}, skipped")}"
|
|
64
|
+
end
|
|
65
|
+
[]
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def warn_moved_tag(names, checkout)
|
|
69
|
+
before = workspace.lock[names.first]['commit']
|
|
70
|
+
return if before == checkout.commit
|
|
71
|
+
|
|
72
|
+
warn("#{names.join(', ')}: the tag #{checkout.resolution.label} moved from #{before.to_s[0, 12]} " \
|
|
73
|
+
"to #{checkout.commit[0, 12]}")
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def change(checkout, name)
|
|
77
|
+
entry = workspace.lock[name]
|
|
78
|
+
candidate = find(checkout, name, entry)
|
|
79
|
+
return unless candidate
|
|
80
|
+
|
|
81
|
+
tree = Tree.read(candidate.path, limits: session.limits)
|
|
82
|
+
folders = workspace.folders(entry)
|
|
83
|
+
folders.each { |folder| workspace.ensure_unedited(name, folder, options[:force]) }
|
|
84
|
+
updated = updated_entry(entry, checkout, tree)
|
|
85
|
+
Change.new(name: name, entry: updated, diff: diff(name, folders, tree, updated), from: entry['commit'],
|
|
86
|
+
to: checkout.commit, source: checkout.source,
|
|
87
|
+
copy: Installer::Copy.new(name: name, tree: tree, targets: folders, entry: updated,
|
|
88
|
+
executable: updated['executable'] ? true : false))
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def find(checkout, name, entry)
|
|
92
|
+
candidate = Discovery.new(checkout.root, subpath: entry['path']).candidates
|
|
93
|
+
.find { |each| each.relative == entry['path'] }
|
|
94
|
+
if candidate&.valid? && candidate.name == name
|
|
95
|
+
return candidate unless checkout.link_inside?(candidate.relative)
|
|
96
|
+
|
|
97
|
+
raise UnsafeError, "#{candidate.relative} holds a symbolic link; skills may not hold links"
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
reason = candidate ? candidate.error || "it is now named #{candidate.name}" : "#{entry['path']} is gone"
|
|
101
|
+
warn("#{name}: cannot update, #{reason}. Run `skill add` again to choose what to install")
|
|
102
|
+
nil
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def updated_entry(entry, checkout, tree)
|
|
106
|
+
entry.merge('commit' => checkout.commit,
|
|
107
|
+
'ref_type' => checkout.resolution ? checkout.resolution.type.to_s : 'local',
|
|
108
|
+
'digest' => tree.digest(executable: entry['executable'] ? true : false))
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Returns what changes from the first installed copy that differs
|
|
112
|
+
# from +tree+, or nothing when every copy already matches it.
|
|
113
|
+
def diff(name, folders, tree, entry)
|
|
114
|
+
executable = entry['executable'] ? true : false
|
|
115
|
+
folders.each do |folder|
|
|
116
|
+
path = Installer.destination(folder, name)
|
|
117
|
+
return tree.diff_from(nil) unless File.directory?(path)
|
|
118
|
+
|
|
119
|
+
installed = Tree.read(path, limits: session.limits)
|
|
120
|
+
unless installed.digest(executable: executable) == entry['digest']
|
|
121
|
+
return tree.diff_from(installed,
|
|
122
|
+
executable: executable)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
NO_DIFF
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def apply(changes)
|
|
129
|
+
return 0 if changes.empty?
|
|
130
|
+
|
|
131
|
+
changes.each { |change| show(change) }
|
|
132
|
+
return 0 if options[:dry_run]
|
|
133
|
+
|
|
134
|
+
changed = changes.reject { |change| change.diff.empty? }
|
|
135
|
+
return refuse if changed.any? && !confirmed?(changed.size)
|
|
136
|
+
|
|
137
|
+
Installer.new.install(changed.map(&:copy))
|
|
138
|
+
changes.each { |change| workspace.lock[change.name] = change.entry }
|
|
139
|
+
workspace.lock.save
|
|
140
|
+
say done("Updated #{changes.map { |change| style.bold(change.name) }.join(', ')}")
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def confirmed?(count)
|
|
144
|
+
options[:yes] || session.confirm?(count == 1 ? 'Apply this update?' : "Apply these #{count} updates?")
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def refuse
|
|
148
|
+
session.error('Nothing changed. Run `skill update --yes` to apply.')
|
|
149
|
+
1
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def show(change)
|
|
153
|
+
range = [change.from, change.to].compact.map { |sha| sha[0, 12] }.join('..')
|
|
154
|
+
say "#{style.bold(change.name)}: #{change.source}#{style.dim(" #{range}") unless range.empty?}"
|
|
155
|
+
say style.dim(" #{compare_url(change)}") if compare_url(change)
|
|
156
|
+
return say(style.dim(' no file changes')) if change.diff.empty?
|
|
157
|
+
|
|
158
|
+
show_files(change.diff)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def show_files(diff)
|
|
162
|
+
{ added: style.green('+'), changed: style.yellow('~'), removed: style.red('-') }.each do |kind, mark|
|
|
163
|
+
diff.public_send(kind).each { |path| say " #{mark} #{path}" }
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def compare_url(change)
|
|
168
|
+
source = change.source
|
|
169
|
+
return unless source.github? && change.from && change.to && change.from != change.to
|
|
170
|
+
|
|
171
|
+
GitHub.compare_url(source.owner, source.repo, change.from, change.to)
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Skill
|
|
4
|
+
# The subcommands of the command line, one class each.
|
|
5
|
+
module Commands
|
|
6
|
+
autoload :Add, 'skill/commands/add'
|
|
7
|
+
autoload :Agents, 'skill/commands/agents'
|
|
8
|
+
autoload :Command, 'skill/commands/command'
|
|
9
|
+
autoload :Find, 'skill/commands/find'
|
|
10
|
+
autoload :List, 'skill/commands/list'
|
|
11
|
+
autoload :Remove, 'skill/commands/remove'
|
|
12
|
+
autoload :Selection, 'skill/commands/selection'
|
|
13
|
+
autoload :Update, 'skill/commands/update'
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'uri'
|
|
5
|
+
|
|
6
|
+
module Skill
|
|
7
|
+
# Searches skills.sh, the directory of Agent Skills vercel-labs/skills
|
|
8
|
+
# reads, through the same public search endpoint its CLI uses.
|
|
9
|
+
# +SKILLS_API_URL+ points it at another server with that endpoint.
|
|
10
|
+
class Directory
|
|
11
|
+
Result = Struct.new(:name, :source, :installs, keyword_init: true) do
|
|
12
|
+
# Returns the command that installs this skill.
|
|
13
|
+
def command
|
|
14
|
+
"skill add #{source}@#{name}"
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
DEFAULT_URL = 'https://skills.sh'
|
|
19
|
+
MAX_BYTES = 4 * 1024 * 1024
|
|
20
|
+
CONTROL = /[\u0000-\u001f\u007f-\u009f---]/
|
|
21
|
+
|
|
22
|
+
def initialize(http: HTTP.new, env: ENV)
|
|
23
|
+
@http = http
|
|
24
|
+
@base = env['SKILLS_API_URL'].to_s.strip.empty? ? DEFAULT_URL : env['SKILLS_API_URL'].strip.chomp('/')
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Returns up to +limit+ Results for +query+, most installed first,
|
|
28
|
+
# optionally only from repositories of +owner+.
|
|
29
|
+
def search(query, owner: nil, limit: 20)
|
|
30
|
+
params = { 'q' => query, 'limit' => limit.to_s }
|
|
31
|
+
params['owner'] = owner if owner
|
|
32
|
+
url = "#{@base}/api/search?#{URI.encode_www_form(params)}"
|
|
33
|
+
response = @http.get(url, limit: MAX_BYTES, headers: { 'Accept' => 'application/json' })
|
|
34
|
+
raise NetworkError, "#{URI(@base).host} answered #{response.status} to the search" unless response.status == 200
|
|
35
|
+
|
|
36
|
+
results(JSON.parse(response.body.force_encoding(Encoding::UTF_8)))
|
|
37
|
+
rescue JSON::ParserError
|
|
38
|
+
raise NetworkError, "#{URI(@base).host} sent a search result that is not JSON"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def results(data)
|
|
44
|
+
skills = data.is_a?(Hash) && data['skills'].is_a?(Array) ? data['skills'] : []
|
|
45
|
+
results = skills.grep(Hash).filter_map do |skill|
|
|
46
|
+
name = clean(skill['skillId'] || skill['name'])
|
|
47
|
+
source = clean(skill['source'])
|
|
48
|
+
next unless Manifest.valid_name?(name) && source.match?(%r{\A[\w.-]+/[\w.-]+(?:/[\w./-]+)?\z})
|
|
49
|
+
|
|
50
|
+
Result.new(name: name, source: source, installs: skill['installs'].to_i)
|
|
51
|
+
end
|
|
52
|
+
results.sort_by { |result| -result.installs }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def clean(value)
|
|
56
|
+
value.to_s.scrub.gsub(CONTROL, '').strip
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Skill
|
|
6
|
+
# Finds the skills in a repository or folder, where vercel-labs/skills
|
|
7
|
+
# looks for them: a +SKILL.md+ at the top, the conventional skill
|
|
8
|
+
# folders (+skills/+, the agents' own folders) up to three levels deep,
|
|
9
|
+
# the plugins of a Claude Code, Codex, or Cursor marketplace file, and,
|
|
10
|
+
# when those hold nothing, the whole tree.
|
|
11
|
+
class Discovery
|
|
12
|
+
# A folder holding a +SKILL.md+. +manifest+ is +nil+ and +error+ says
|
|
13
|
+
# why when the +SKILL.md+ is invalid.
|
|
14
|
+
Candidate = Struct.new(:path, :relative, :manifest, :error, :plugin, keyword_init: true) do
|
|
15
|
+
def valid?
|
|
16
|
+
!manifest.nil?
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def name
|
|
20
|
+
manifest ? manifest.name : File.basename(relative)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
CONTAINERS = %w[skills skills/.curated skills/.experimental skills/.system].freeze
|
|
25
|
+
MARKETPLACES = %w[.claude-plugin/marketplace.json .agents/plugins/marketplace.json
|
|
26
|
+
.cursor-plugin/marketplace.json].freeze
|
|
27
|
+
PLUGINS = %w[.claude-plugin/plugin.json .codex-plugin/plugin.json .cursor-plugin/plugin.json].freeze
|
|
28
|
+
SKIPPED = %w[.git node_modules vendor/bundle].freeze
|
|
29
|
+
CONTAINER_DEPTH = 3
|
|
30
|
+
FULL_DEPTH = 8
|
|
31
|
+
|
|
32
|
+
# External plugins a marketplace lists in other repositories, which
|
|
33
|
+
# need a +skill add+ of their own.
|
|
34
|
+
attr_reader :external, :warnings
|
|
35
|
+
|
|
36
|
+
def initialize(root, subpath: '', full_depth: false)
|
|
37
|
+
@root = File.realpath(root.to_s)
|
|
38
|
+
@base = inside!(File.join(@root, Source.clean_path(subpath)))
|
|
39
|
+
@full_depth = full_depth
|
|
40
|
+
@found = {}
|
|
41
|
+
@external = []
|
|
42
|
+
@warnings = []
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Returns every Candidate, ordered by name.
|
|
46
|
+
def candidates
|
|
47
|
+
@candidates ||= begin
|
|
48
|
+
search
|
|
49
|
+
@found.values.sort_by { |candidate| [candidate.name, candidate.relative] }
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def search
|
|
56
|
+
raise NotFoundError, "#{relative(@base)} is not a folder" unless File.directory?(@base) && !File.symlink?(@base)
|
|
57
|
+
return add(@base) if skill?(@base)
|
|
58
|
+
|
|
59
|
+
plugins
|
|
60
|
+
(CONTAINERS + Agents.all.map(&:project_dir)).uniq.each do |container|
|
|
61
|
+
walk(File.join(@base, container), CONTAINER_DEPTH)
|
|
62
|
+
end
|
|
63
|
+
walk(@base, FULL_DEPTH) if @found.empty? || @full_depth
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def walk(dir, depth, plugin: nil)
|
|
67
|
+
return unless File.directory?(dir) && !File.symlink?(dir)
|
|
68
|
+
return add(dir, plugin: plugin) if skill?(dir)
|
|
69
|
+
return if depth.zero?
|
|
70
|
+
|
|
71
|
+
Dir.children(dir).sort.each do |child|
|
|
72
|
+
next if SKIPPED.include?(child) || SKIPPED.include?(relative(File.join(dir, child)))
|
|
73
|
+
|
|
74
|
+
walk(File.join(dir, child), depth - 1, plugin: plugin)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def skill?(dir)
|
|
79
|
+
file = File.join(dir, Manifest::FILE)
|
|
80
|
+
File.file?(file) && !File.symlink?(file)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def add(dir, plugin: nil)
|
|
84
|
+
key = relative(dir)
|
|
85
|
+
return if @found.key?(key)
|
|
86
|
+
|
|
87
|
+
manifest = Manifest.load(dir)
|
|
88
|
+
if (other = @found.values.find { |candidate| candidate.valid? && candidate.name == manifest.name })
|
|
89
|
+
@warnings << "Two skills are named #{manifest.name}: #{other.relative} and #{key}. Using #{other.relative}"
|
|
90
|
+
return
|
|
91
|
+
end
|
|
92
|
+
@found[key] = Candidate.new(path: dir, relative: key, manifest: manifest, plugin: plugin)
|
|
93
|
+
rescue InvalidSkillError => e
|
|
94
|
+
@found[key] = Candidate.new(path: dir, relative: key, error: e.message.delete_prefix("#{@root}/"), plugin: plugin)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def plugins
|
|
98
|
+
MARKETPLACES.each { |file| marketplace(File.join(@base, file)) }
|
|
99
|
+
PLUGINS.each do |file|
|
|
100
|
+
data = json(File.join(@base, file))
|
|
101
|
+
plugin_skills(@base, data['skills'], data['name']) if data
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def marketplace(file)
|
|
106
|
+
data = json(file)
|
|
107
|
+
return unless data && data['plugins'].is_a?(Array)
|
|
108
|
+
|
|
109
|
+
root = data.dig('metadata', 'pluginRoot').to_s
|
|
110
|
+
data['plugins'].grep(Hash).each { |entry| marketplace_plugin(entry, root) }
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def marketplace_plugin(entry, plugin_root)
|
|
114
|
+
name = entry['name'].to_s
|
|
115
|
+
source = entry['source']
|
|
116
|
+
source = source['path'] if source.is_a?(Hash) && [nil, 'local'].include?(source['source'])
|
|
117
|
+
unless source.nil? || source.is_a?(String)
|
|
118
|
+
@external << { 'plugin' => name, 'source' => external_source(source) }
|
|
119
|
+
return
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
path = source.to_s
|
|
123
|
+
path = File.join(plugin_root, path) unless plugin_root.empty? || path.start_with?('./', '../')
|
|
124
|
+
plugin_skills(File.join(@base, Source.clean_path(path)), entry['skills'], name)
|
|
125
|
+
rescue UnsafeError
|
|
126
|
+
@warnings << "The #{name} plugin points outside the repository; skipped"
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def plugin_skills(plugin_dir, skills, name)
|
|
130
|
+
plugin_dir = inside!(plugin_dir)
|
|
131
|
+
paths = skills.is_a?(Array) ? skills.grep(String) : []
|
|
132
|
+
return walk(File.join(plugin_dir, 'skills'), CONTAINER_DEPTH, plugin: name) if paths.empty?
|
|
133
|
+
|
|
134
|
+
paths.each { |path| walk(inside!(File.join(plugin_dir, Source.clean_path(path))), 1, plugin: name) }
|
|
135
|
+
rescue UnsafeError
|
|
136
|
+
@warnings << "The #{name} plugin points outside the repository; skipped"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def external_source(source)
|
|
140
|
+
repo = source['repo'] || source['url'].to_s[%r{github\.com[/:]([\w.-]+/[\w.-]+?)(?:\.git)?/?\z}, 1]
|
|
141
|
+
repo ? [repo, source['path']].compact.join('/') : source['url'] || source['source']
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def json(file)
|
|
145
|
+
return unless File.file?(file) && !File.symlink?(file)
|
|
146
|
+
|
|
147
|
+
data = JSON.parse(File.read(file, encoding: 'UTF-8').delete_prefix("\u{FEFF}"))
|
|
148
|
+
data.is_a?(Hash) ? data : nil
|
|
149
|
+
rescue JSON::ParserError
|
|
150
|
+
@warnings << "#{relative(file)} is not valid JSON; skipped"
|
|
151
|
+
nil
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def inside!(path)
|
|
155
|
+
expanded = File.expand_path(path)
|
|
156
|
+
expanded = File.realpath(expanded) if File.exist?(expanded)
|
|
157
|
+
return expanded if expanded == @root || expanded.start_with?("#{@root}/")
|
|
158
|
+
|
|
159
|
+
raise UnsafeError, "#{path} leaves #{@root}"
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def relative(path)
|
|
163
|
+
path == @root ? '.' : path.delete_prefix("#{@root}/")
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'tmpdir'
|
|
4
|
+
|
|
5
|
+
module Skill
|
|
6
|
+
# Gives a Source's files as a folder: a local directory in place, or a
|
|
7
|
+
# GitHub commit downloaded and extracted into a temporary folder.
|
|
8
|
+
class Fetcher
|
|
9
|
+
# +root+ is the folder, +resolution+ a GitHub::Resolution (or +nil+ for a
|
|
10
|
+
# local directory), and +links+ the archive paths that were links.
|
|
11
|
+
Checkout = Struct.new(:source, :root, :resolution, :links, keyword_init: true) do
|
|
12
|
+
def commit
|
|
13
|
+
resolution&.commit
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Returns whether the archive held a link inside the folder +relative+.
|
|
17
|
+
def link_inside?(relative)
|
|
18
|
+
prefix = relative == '.' ? '' : "#{relative}/"
|
|
19
|
+
links.any? { |link| link == relative || link.start_with?(prefix) }
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Downloads go into folders under +workdir+, which the caller deletes.
|
|
24
|
+
def initialize(github:, limits:, workdir:)
|
|
25
|
+
@github = github
|
|
26
|
+
@limits = limits
|
|
27
|
+
@workdir = workdir
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Resolves +source+'s ref.
|
|
31
|
+
def resolve(source)
|
|
32
|
+
@github.resolve(source.owner, source.repo, source.ref)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Returns a Checkout of +source+ at +resolution+ (resolved when +nil+).
|
|
36
|
+
def checkout(source, resolution: nil)
|
|
37
|
+
return local(source) if source.local?
|
|
38
|
+
|
|
39
|
+
resolution ||= resolve(source)
|
|
40
|
+
directory = Dir.mktmpdir('checkout-', @workdir)
|
|
41
|
+
archive = File.join(directory, 'source.tar.gz')
|
|
42
|
+
File.open(archive, 'wb') { |io| @github.download(source.owner, source.repo, resolution.commit, io) }
|
|
43
|
+
root = File.join(directory, 'source')
|
|
44
|
+
Dir.mkdir(root)
|
|
45
|
+
result = Archive.new(limits: @limits).extract(archive, root)
|
|
46
|
+
File.delete(archive)
|
|
47
|
+
if result.commit && result.commit != resolution.commit
|
|
48
|
+
raise NetworkError, "GitHub sent commit #{result.commit[0, 12]} for #{resolution.commit[0, 12]}"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
Checkout.new(source: source, root: root, resolution: resolution, links: result.links)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def local(source)
|
|
57
|
+
raise NotFoundError, "No folder at #{source.path}" unless File.directory?(source.path)
|
|
58
|
+
|
|
59
|
+
Checkout.new(source: source, root: File.realpath(source.path), resolution: nil, links: [])
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
data/lib/skill/github.rb
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Skill
|
|
4
|
+
# Reads GitHub repositories without the API and without git: refs come
|
|
5
|
+
# from git's smart HTTP advertisement, files from one tarball of a commit.
|
|
6
|
+
# Neither counts against the API's rate limit. +GITHUB_TOKEN+ or
|
|
7
|
+
# +GH_TOKEN+, when set, opens private repositories.
|
|
8
|
+
class GitHub
|
|
9
|
+
# A ref resolved to a commit. +type+ is +:branch+, +:tag+, or +:commit+;
|
|
10
|
+
# +name+ is +nil+ for the repository's default branch.
|
|
11
|
+
Resolution = Struct.new(:type, :name, :commit, :branch, keyword_init: true) do
|
|
12
|
+
def pinned?
|
|
13
|
+
type == :commit
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Returns the ref as people read it: the tag, the branch, or the
|
|
17
|
+
# default branch's name.
|
|
18
|
+
def label
|
|
19
|
+
name || branch || 'HEAD'
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
SHA = /\A\h{40}\z/
|
|
24
|
+
MAX_REFS_BYTES = 32 * 1024 * 1024
|
|
25
|
+
|
|
26
|
+
def initialize(http: HTTP.new, env: ENV, limits: Limits.new(env))
|
|
27
|
+
@http = http
|
|
28
|
+
@token = [env['GITHUB_TOKEN'], env['GH_TOKEN']].map(&:to_s).find { |value| !value.strip.empty? }&.strip
|
|
29
|
+
@limits = limits
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Resolves +ref+ (a branch, a tag, a full commit SHA, or +nil+ for the
|
|
33
|
+
# default branch) in +owner/repo+. A tag wins over a branch of the same
|
|
34
|
+
# name, as in git.
|
|
35
|
+
def resolve(owner, repo, ref)
|
|
36
|
+
return Resolution.new(type: :commit, name: ref.downcase, commit: ref.downcase) if ref&.match?(SHA)
|
|
37
|
+
|
|
38
|
+
refs = advertised_refs(owner, repo)
|
|
39
|
+
return default_branch(owner, repo, refs) if ref.nil?
|
|
40
|
+
|
|
41
|
+
name = ref.delete_prefix('refs/heads/').delete_prefix('refs/tags/')
|
|
42
|
+
if (commit = refs[:tags][name]) && !ref.start_with?('refs/heads/')
|
|
43
|
+
Resolution.new(type: :tag, name: name, commit: commit)
|
|
44
|
+
elsif (commit = refs[:branches][name])
|
|
45
|
+
Resolution.new(type: :branch, name: name, commit: commit, branch: name)
|
|
46
|
+
elsif ref.match?(/\A\h{4,39}\z/)
|
|
47
|
+
raise NotFoundError,
|
|
48
|
+
"#{ref} is not a branch or tag of #{owner}/#{repo}. Pin commits by their full 40-character SHA"
|
|
49
|
+
else
|
|
50
|
+
raise NotFoundError, "#{owner}/#{repo} has no branch or tag named #{ref}"
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Downloads +commit+ of +owner/repo+ as a gzipped tarball into +io+.
|
|
55
|
+
def download(owner, repo, commit, io)
|
|
56
|
+
url, headers = archive_request(owner, repo, commit)
|
|
57
|
+
response = download_archive(url, headers, io, owner, repo)
|
|
58
|
+
return if response.status == 200
|
|
59
|
+
|
|
60
|
+
if [404, 422].include?(response.status)
|
|
61
|
+
raise NotFoundError, "#{owner}/#{repo} has no commit #{commit}#{private_hint}"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
raise NetworkError, "GitHub answered #{response.status} for the archive of #{owner}/#{repo}"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Returns a link comparing two commits, for showing what changed.
|
|
68
|
+
def self.compare_url(owner, repo, from, to)
|
|
69
|
+
"https://github.com/#{owner}/#{repo}/compare/#{from[0, 12]}...#{to[0, 12]}"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def download_archive(url, headers, io, owner, repo)
|
|
75
|
+
@http.get(url, headers: headers, limit: @limits.download_bytes, io: io)
|
|
76
|
+
rescue TooLargeError
|
|
77
|
+
message = "#{owner}/#{repo} is larger than #{@limits.download_bytes} bytes to download"
|
|
78
|
+
raise TooLargeError, Limits.exceeded(:download_bytes, message)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def advertised_refs(owner, repo)
|
|
82
|
+
response = @http.get("https://github.com/#{owner}/#{repo}.git/info/refs?service=git-upload-pack",
|
|
83
|
+
headers: basic_auth, limit: MAX_REFS_BYTES)
|
|
84
|
+
if [401, 403, 404].include?(response.status)
|
|
85
|
+
raise NotFoundError, "No GitHub repository #{owner}/#{repo}#{private_hint}"
|
|
86
|
+
end
|
|
87
|
+
raise NetworkError, "GitHub answered #{response.status} for #{owner}/#{repo}" unless response.status == 200
|
|
88
|
+
|
|
89
|
+
parse_refs(response.body)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Reads git's pkt-line ref advertisement: a service line, a flush, then
|
|
93
|
+
# one "<sha> <ref>" line per ref, the first carrying capabilities after
|
|
94
|
+
# a NUL byte. Peeled tags end in ^{}.
|
|
95
|
+
def parse_refs(body)
|
|
96
|
+
refs = { head: nil, symref: nil, branches: {}, tags: {} }
|
|
97
|
+
pkt_lines(body).each do |line|
|
|
98
|
+
line, capabilities = line.chomp.split("\0", 2)
|
|
99
|
+
refs[:symref] ||= capabilities.to_s[%r{symref=HEAD:refs/heads/(\S+)}, 1]
|
|
100
|
+
sha, name = line.split(' ', 2)
|
|
101
|
+
next unless sha&.match?(SHA) && name
|
|
102
|
+
|
|
103
|
+
add_ref(refs, sha, name)
|
|
104
|
+
end
|
|
105
|
+
refs
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def add_ref(refs, sha, name)
|
|
109
|
+
case name
|
|
110
|
+
when 'HEAD' then refs[:head] = sha
|
|
111
|
+
when %r{\Arefs/heads/(.+)\z} then refs[:branches][Regexp.last_match(1)] = sha
|
|
112
|
+
when %r{\Arefs/tags/(.+)\^\{\}\z} then refs[:tags][Regexp.last_match(1)] = sha
|
|
113
|
+
when %r{\Arefs/tags/(.+)\z} then refs[:tags][Regexp.last_match(1)] ||= sha
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def pkt_lines(body)
|
|
118
|
+
lines = []
|
|
119
|
+
position = 0
|
|
120
|
+
while position + 4 <= body.bytesize
|
|
121
|
+
length = Integer(body.byteslice(position, 4), 16)
|
|
122
|
+
position += 4
|
|
123
|
+
next if length.zero?
|
|
124
|
+
raise NetworkError, 'GitHub sent a malformed ref list' if length < 4
|
|
125
|
+
|
|
126
|
+
lines << body.byteslice(position, length - 4).force_encoding(Encoding::UTF_8).scrub
|
|
127
|
+
position += length - 4
|
|
128
|
+
end
|
|
129
|
+
lines.reject { |line| line.start_with?('# service=') }
|
|
130
|
+
rescue ArgumentError
|
|
131
|
+
raise NetworkError, 'GitHub sent a malformed ref list'
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def default_branch(owner, repo, refs)
|
|
135
|
+
raise NotFoundError, "#{owner}/#{repo} is empty" unless refs[:head]
|
|
136
|
+
|
|
137
|
+
branch = refs[:symref] || refs[:branches].key(refs[:head])
|
|
138
|
+
Resolution.new(type: :branch, name: nil, commit: refs[:head], branch: branch)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def archive_request(owner, repo, commit)
|
|
142
|
+
return ["https://codeload.github.com/#{owner}/#{repo}/tar.gz/#{commit}", {}] unless @token
|
|
143
|
+
|
|
144
|
+
["https://api.github.com/repos/#{owner}/#{repo}/tarball/#{commit}",
|
|
145
|
+
{ 'Authorization' => "Bearer #{@token}", 'Accept' => 'application/vnd.github+json' }]
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def basic_auth
|
|
149
|
+
return {} unless @token
|
|
150
|
+
|
|
151
|
+
{ 'Authorization' => "Basic #{["x-access-token:#{@token}"].pack('m0')}" }
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def private_hint
|
|
155
|
+
@token ? '' : ' (for a private repository, set GITHUB_TOKEN)'
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|