gitlab-labkit 5.6.1 → 5.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/hooks/precommit-validate.sh +1 -0
- data/.copier-answers.yml +1 -1
- data/.gitlab/CODEOWNERS +1 -1
- data/.gitlab-ci.yml +17 -3
- data/.opencode/plugins/precommit-validate.ts +181 -56
- data/.pre-commit-config.yaml +1 -1
- data/AGENTS.md +34 -38
- data/CODEOWNERS +1 -1
- data/CONTRIBUTING.md +1 -1
- data/lib/labkit/fields.rb +14 -0
- data/lib/labkit/rate_limit/README.md +50 -0
- data/lib/labkit/rate_limit/block_summary.rb +119 -0
- data/lib/labkit/rate_limit.rb +1 -0
- data/scripts/prepare-dev-env.sh +1 -0
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bbffb0883d80c949bfb28ffec2943b9cc89ffb88c2c02656afe99ad777e0f152
|
|
4
|
+
data.tar.gz: 93610eb46b3dbbfebe40c1b97d31d07b69af3f5e5ee0f152be29a66eca8da03e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c01235e9277e28adbde6cb2e14cedebbabe78b46a8f7e771664875da94cd59ea3c31e2033b152ea8e23dce3cd8ce84c0c5fd310c9691d1662006df0e87f66b67
|
|
7
|
+
data.tar.gz: 87c05f1a0543e7969b35023d227d32cabc87455caa99aeca54a83832a5e40a2968895afb3982c4ecfcadc31d891558ce4abccb249f0758f95903367fd56b82df
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
# touched. On failure it prints the output to stderr and exits 2, which feeds the
|
|
6
6
|
# result back to Claude so it can fix the issue. A clean run exits 0 silently.
|
|
7
7
|
set -euo pipefail
|
|
8
|
+
IFS=$'\n\t'
|
|
8
9
|
|
|
9
10
|
# no-commit-to-branch guards the branch, not file content, and would fire
|
|
10
11
|
# spuriously (e.g. on jj's detached HEAD). It is irrelevant to file validation.
|
data/.copier-answers.yml
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# See the project for instructions on how to update the project
|
|
4
4
|
#
|
|
5
5
|
# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY
|
|
6
|
-
_commit: v1.
|
|
6
|
+
_commit: v1.89.0
|
|
7
7
|
_src_path: https://gitlab.com/gitlab-com/gl-infra/common-template-copier.git
|
|
8
8
|
ee_licensed: false
|
|
9
9
|
gitlab_namespace: gitlab-org/ruby/gems
|
data/.gitlab/CODEOWNERS
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
* @
|
|
1
|
+
* @mkaeppler @ayufan @hmerscher @sankalp_gl @hardikgala @nindurkar @ashs2
|
data/.gitlab-ci.yml
CHANGED
|
@@ -19,13 +19,27 @@ include:
|
|
|
19
19
|
# It includes standard checks, gitlab-scanners, validations and release processes
|
|
20
20
|
# common to all projects using this template library.
|
|
21
21
|
# see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/templates/standard.md
|
|
22
|
-
- component: $CI_SERVER_FQDN/gitlab-com/gl-infra/common-ci-tasks/standard-build@v5.
|
|
22
|
+
- component: $CI_SERVER_FQDN/gitlab-com/gl-infra/common-ci-tasks/standard-build@v5.46
|
|
23
23
|
|
|
24
24
|
# Runs rspec tests and rubocop on the project
|
|
25
25
|
# see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/templates/ruby.md
|
|
26
|
-
- component: $CI_SERVER_FQDN/gitlab-com/gl-infra/common-ci-tasks/ruby-build@v5.
|
|
26
|
+
- component: $CI_SERVER_FQDN/gitlab-com/gl-infra/common-ci-tasks/ruby-build@v5.46
|
|
27
27
|
|
|
28
|
-
- component: $CI_SERVER_FQDN/gitlab-com/gl-infra/common-ci-tasks/danger@v5.
|
|
28
|
+
- component: $CI_SERVER_FQDN/gitlab-com/gl-infra/common-ci-tasks/danger@v5.46
|
|
29
|
+
|
|
30
|
+
# danger-review reads its token from Vault, which the community fork has no
|
|
31
|
+
# access to, so it fails on merge requests from the fork. Let it fail for those;
|
|
32
|
+
# the second rule is the component's own. See
|
|
33
|
+
# https://gitlab.com/gitlab-org/labkit/-/work_items/171
|
|
34
|
+
danger-review:
|
|
35
|
+
rules:
|
|
36
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == "gitlab-community/gitlab-org/ruby/gems/labkit-ruby"
|
|
37
|
+
exists:
|
|
38
|
+
- Dangerfile
|
|
39
|
+
allow_failure: true
|
|
40
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
41
|
+
exists:
|
|
42
|
+
- Dangerfile
|
|
29
43
|
|
|
30
44
|
# Attach a redis service to the rspec job from common-ci-tasks/ruby-build.
|
|
31
45
|
# GitLab merges keys when a local job has the same name as an included one,
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Hooks, PluginModule } from "@opencode-ai/plugin";
|
|
2
|
+
import type { Plugin } from "@opencode/plugin";
|
|
3
|
+
import { execFile, spawn } from "node:child_process";
|
|
2
4
|
import { existsSync } from "node:fs";
|
|
3
5
|
import { isAbsolute, join, resolve } from "node:path";
|
|
4
6
|
|
|
@@ -22,80 +24,203 @@ import { isAbsolute, join, resolve } from "node:path";
|
|
|
22
24
|
*
|
|
23
25
|
* The script is reused verbatim: it reads Claude Code's PostToolUse payload on
|
|
24
26
|
* stdin (`{ "tool_input": { "file_path": "..." } }`), so we synthesize that same
|
|
25
|
-
* shape here from OpenCode's tool
|
|
27
|
+
* shape here from OpenCode's tool input.
|
|
28
|
+
*
|
|
29
|
+
* Supports both plugin APIs from one default export, as described in
|
|
30
|
+
* https://opencode.ai/v2/docs/build/plugins/migrate-v1:
|
|
31
|
+
* - OpenCode 2 reads `id` and `setup()` (the `@opencode/plugin` API).
|
|
32
|
+
* - OpenCode 1 (>= 1.18.29) calls `server()` (the `@opencode-ai/plugin` API).
|
|
33
|
+
* Both imports are type-only, so neither OpenCode version needs the other's
|
|
34
|
+
* plugin package installed at runtime. Remove `server()` once V1 is gone.
|
|
26
35
|
*
|
|
27
36
|
* Auto-loaded from `.opencode/plugins/`.
|
|
28
37
|
*/
|
|
29
38
|
|
|
30
|
-
|
|
31
|
-
// matcher. Current OpenCode may use `apply_patch` for file edits.
|
|
32
|
-
const FILE_TOOLS = new Set(["apply_patch", "edit", "write"]);
|
|
39
|
+
const ID = "precommit-validate";
|
|
33
40
|
|
|
34
41
|
const HOOK_SCRIPT = ".claude/hooks/precommit-validate.sh";
|
|
35
42
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
43
|
+
// OpenCode file-mutating tools, analogous to Claude Code's `Edit|Write|MultiEdit`
|
|
44
|
+
// matcher. V1 and V2 both have `edit` and `write`; patches go through
|
|
45
|
+
// `apply_patch` in V1 and `patch` in V2.
|
|
46
|
+
const FILE_TOOLS = new Set(["apply_patch", "edit", "patch", "write"]);
|
|
39
47
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
48
|
+
interface HookResult {
|
|
49
|
+
/** Exit code 2: pre-commit findings to feed back to the model. */
|
|
50
|
+
readonly reports: string[];
|
|
51
|
+
/** Any other failure: environment problems to log for the user. */
|
|
52
|
+
readonly warnings: string[];
|
|
53
|
+
}
|
|
45
54
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (filePath) filePaths.add(filePath);
|
|
55
|
+
const asRecord = (value: unknown): Record<string, unknown> =>
|
|
56
|
+
value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
|
49
57
|
|
|
50
|
-
|
|
51
|
-
if (typeof file.filePath === "string") filePaths.add(file.filePath);
|
|
52
|
-
}
|
|
58
|
+
const asString = (value: unknown): string | undefined => (typeof value === "string" && value ? value : undefined);
|
|
53
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Collect the files a tool call touched from its input and result metadata.
|
|
62
|
+
*
|
|
63
|
+
* Input: V2 `edit`/`write` take `path`; V1 takes `filePath`.
|
|
64
|
+
* Metadata: `files` entries carry `file` in V2 and `filePath` in V1. This is
|
|
65
|
+
* the only source for patch tools, whose input is a single `patchText` blob.
|
|
66
|
+
*/
|
|
67
|
+
function touchedFiles(input: unknown, metadata: unknown): Set<string> {
|
|
68
|
+
const args = asRecord(input);
|
|
69
|
+
const files = new Set<string>();
|
|
70
|
+
|
|
71
|
+
const direct = asString(args.path) ?? asString(args.filePath);
|
|
72
|
+
if (direct) files.add(direct);
|
|
73
|
+
|
|
74
|
+
const entries = asRecord(metadata).files;
|
|
75
|
+
for (const entry of Array.isArray(entries) ? entries : []) {
|
|
76
|
+
const file = asRecord(entry);
|
|
77
|
+
// A deleted file has nothing left to validate.
|
|
78
|
+
if (file.status === "deleted" || file.type === "delete") continue;
|
|
79
|
+
const path = asString(file.file) ?? asString(file.filePath);
|
|
80
|
+
if (path) files.add(path);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return files;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Run the shared hook script for one file, feeding the payload on stdin. */
|
|
87
|
+
function runHookScript(
|
|
88
|
+
script: string,
|
|
89
|
+
root: string,
|
|
90
|
+
payload: string,
|
|
91
|
+
signal?: AbortSignal,
|
|
92
|
+
): Promise<{ code: number | null; stdout: string; stderr: string }> {
|
|
93
|
+
return new Promise((done, fail) => {
|
|
94
|
+
const child = spawn(script, [], { cwd: root, signal, stdio: ["pipe", "pipe", "pipe"] });
|
|
95
|
+
let stdout = "";
|
|
96
|
+
let stderr = "";
|
|
97
|
+
child.stdout.setEncoding("utf8").on("data", (chunk: string) => (stdout += chunk));
|
|
98
|
+
child.stderr.setEncoding("utf8").on("data", (chunk: string) => (stderr += chunk));
|
|
99
|
+
// The script exits early (e.g. mise missing) without reading stdin.
|
|
100
|
+
child.stdin.on("error", () => {});
|
|
101
|
+
child.on("error", fail);
|
|
102
|
+
child.on("close", (code) => done({ code, stdout, stderr }));
|
|
103
|
+
child.stdin.end(payload);
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Validate every touched file and sort the outcomes into reports and warnings. */
|
|
108
|
+
async function validate(root: string, filePaths: Iterable<string>, signal?: AbortSignal): Promise<HookResult> {
|
|
109
|
+
const result: HookResult = { reports: [], warnings: [] };
|
|
110
|
+
const script = join(root, HOOK_SCRIPT);
|
|
111
|
+
// Inert when the shared script is absent (e.g. loaded in another repo).
|
|
112
|
+
if (!existsSync(script)) return result;
|
|
113
|
+
|
|
114
|
+
for (const filePath of filePaths) {
|
|
115
|
+
if (signal?.aborted) break;
|
|
116
|
+
const absolute = isAbsolute(filePath) ? filePath : resolve(root, filePath);
|
|
117
|
+
|
|
118
|
+
// Same JSON shape Claude Code's PostToolUse hook delivers on stdin. The
|
|
119
|
+
// script itself decides whether the file lives inside the repo.
|
|
120
|
+
const payload = JSON.stringify({ tool_input: { file_path: absolute } });
|
|
121
|
+
|
|
122
|
+
let run: Awaited<ReturnType<typeof runHookScript>>;
|
|
123
|
+
try {
|
|
124
|
+
run = await runHookScript(script, root, payload, signal);
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (signal?.aborted) break; // plugin unloaded mid-run
|
|
127
|
+
result.warnings.push(`pre-commit validation hook failed to start: ${String(error)}`);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (run.code === 0) continue; // clean / no-op
|
|
132
|
+
|
|
133
|
+
const report = run.stderr.trim() || run.stdout.trim();
|
|
134
|
+
|
|
135
|
+
if (run.code === 2) {
|
|
136
|
+
// Blocking: pre-commit reported issues. Surface them to the model, the
|
|
137
|
+
// OpenCode equivalent of feeding the hook's stderr back to Claude Code.
|
|
138
|
+
if (report) result.reports.push(report);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Non-blocking error (e.g. exit 1 = mise not on PATH). Not something the
|
|
143
|
+
// model can act on, so log it for the user instead of polluting the turn.
|
|
144
|
+
result.warnings.push(report || `pre-commit validation hook exited ${run.code}`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return result;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const formatReports = (reports: string[]): string =>
|
|
151
|
+
`<pre-commit-validation>\n${reports.join("\n\n")}\n</pre-commit-validation>`;
|
|
152
|
+
|
|
153
|
+
/** The repository root, so the hook script is found from any subdirectory. */
|
|
154
|
+
function gitToplevel(directory: string): Promise<string> {
|
|
155
|
+
return new Promise((done) => {
|
|
156
|
+
execFile("git", ["rev-parse", "--show-toplevel"], { cwd: directory }, (error, stdout) => {
|
|
157
|
+
done(error ? directory : stdout.trim() || directory);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** OpenCode 2 implementation (`@opencode/plugin`). */
|
|
163
|
+
const v2: Plugin.Plugin = {
|
|
164
|
+
id: ID,
|
|
165
|
+
async setup(ctx) {
|
|
166
|
+
const root = await gitToplevel(ctx.location.directory);
|
|
167
|
+
// Hook events carry no abort signal; stop in-flight runs on unload instead.
|
|
168
|
+
const controller = new AbortController();
|
|
169
|
+
|
|
170
|
+
await ctx.tool.hook("execute.after", async (event) => {
|
|
171
|
+
if (event.status !== "completed") return;
|
|
172
|
+
if (!FILE_TOOLS.has(event.tool)) return;
|
|
173
|
+
|
|
174
|
+
const filePaths = touchedFiles(event.input, event.result.metadata);
|
|
54
175
|
if (filePaths.size === 0) return;
|
|
55
176
|
|
|
56
|
-
const reports
|
|
177
|
+
const { reports, warnings } = await validate(root, filePaths, controller.signal);
|
|
178
|
+
|
|
179
|
+
for (const warning of warnings) console.warn(`[${ID}] ${warning}`);
|
|
57
180
|
|
|
58
|
-
|
|
59
|
-
const absolute = isAbsolute(filePath) ? filePath : resolve(root, filePath);
|
|
181
|
+
if (reports.length === 0) return;
|
|
60
182
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
183
|
+
// Results are read-only: replace the result, appending to its content.
|
|
184
|
+
const text = formatReports(reports);
|
|
185
|
+
const content = event.result.content;
|
|
186
|
+
event.result = {
|
|
187
|
+
...event.result,
|
|
188
|
+
content:
|
|
189
|
+
typeof content === "string" || content === undefined
|
|
190
|
+
? content ? `${content}\n\n${text}` : text
|
|
191
|
+
: [...content, { type: "text", text }],
|
|
192
|
+
};
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
return () => controller.abort();
|
|
196
|
+
},
|
|
197
|
+
};
|
|
64
198
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
199
|
+
/** OpenCode 1 implementation (`@opencode-ai/plugin`, >= 1.18.29). */
|
|
200
|
+
const v1: PluginModule = {
|
|
201
|
+
id: ID,
|
|
202
|
+
async server({ client, directory, worktree }) {
|
|
203
|
+
const root = worktree || directory;
|
|
69
204
|
|
|
70
|
-
|
|
205
|
+
const hooks: Hooks = {
|
|
206
|
+
"tool.execute.after": async (input, output) => {
|
|
207
|
+
if (!FILE_TOOLS.has(input.tool)) return;
|
|
71
208
|
|
|
72
|
-
const
|
|
73
|
-
|
|
209
|
+
const filePaths = touchedFiles(input.args, output.metadata);
|
|
210
|
+
if (filePaths.size === 0) return;
|
|
74
211
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
if (report) reports.push(report);
|
|
80
|
-
continue;
|
|
212
|
+
const { reports, warnings } = await validate(root, filePaths);
|
|
213
|
+
|
|
214
|
+
for (const warning of warnings) {
|
|
215
|
+
await client.app.log({ body: { service: ID, level: "warn", message: warning } });
|
|
81
216
|
}
|
|
82
217
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
message: report || `pre-commit validation hook exited ${result.exitCode}`,
|
|
90
|
-
},
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
if (reports.length > 0) {
|
|
95
|
-
output.output += `\n\n<pre-commit-validation>\n${reports.join("\n\n")}\n</pre-commit-validation>`;
|
|
96
|
-
}
|
|
97
|
-
},
|
|
98
|
-
};
|
|
99
|
-
|
|
100
|
-
return hooks;
|
|
218
|
+
if (reports.length > 0) output.output += `\n\n${formatReports(reports)}`;
|
|
219
|
+
},
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
return hooks;
|
|
223
|
+
},
|
|
101
224
|
};
|
|
225
|
+
|
|
226
|
+
export default { ...v2, server: v1.server };
|
data/.pre-commit-config.yaml
CHANGED
|
@@ -25,7 +25,7 @@ repos:
|
|
|
25
25
|
# Documentation available at
|
|
26
26
|
# https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docs/pre-commit.md
|
|
27
27
|
- repo: https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks
|
|
28
|
-
rev: v5.
|
|
28
|
+
rev: v5.46 # renovate:managed
|
|
29
29
|
|
|
30
30
|
hooks:
|
|
31
31
|
- id: shellcheck # Run shellcheck for changed Shell files
|
data/AGENTS.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
# Agent guidelines
|
|
2
|
+
|
|
3
|
+
This project uses [Conventional Commits](https://www.conventionalcommits.org/).
|
|
4
|
+
Commit messages drive automated [semantic releases](https://semantic-release.gitbook.io/)
|
|
5
|
+
(see `.releaserc.json`), so use `feat:`, `fix:`, etc. prefixes to ensure the
|
|
6
|
+
correct version bump and changelog entry.
|
|
7
|
+
|
|
8
|
+
## AI-authored GitLab comments
|
|
9
|
+
|
|
10
|
+
Wrap the full body of any comment, note, or reply you post to GitLab (issues,
|
|
11
|
+
merge requests, epics — via `glab`/`glab api`) in `<:robot:>` / `</:robot:>`
|
|
12
|
+
tags (GitLab emoji shortcode, renders as 🤖), so readers can tell at a glance
|
|
13
|
+
that it's AI-generated:
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
<:robot:>
|
|
17
|
+
|
|
18
|
+
...full comment body...
|
|
19
|
+
|
|
20
|
+
</:robot:>
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
This applies to newly posted comments. It does not apply to code, commit
|
|
24
|
+
messages, or MR/issue descriptions unless asked.
|
|
25
|
+
|
|
26
|
+
<!--
|
|
27
|
+
####################################################################
|
|
28
|
+
# 🌊✨ PLEASE ADD PROJECT SPECIFIC RULES BELOW THIS LINE ✨🌊 #
|
|
29
|
+
# The section above is managed by the common-template-copier #
|
|
30
|
+
# template and is updated automatically on `copier update`. #
|
|
31
|
+
# Add your project-specific instructions below this line so #
|
|
32
|
+
# they are preserved when the template is updated. #
|
|
33
|
+
####################################################################
|
|
34
|
+
-->
|
|
3
35
|
|
|
4
36
|
Instructions for AI agents and automated tooling working in this repository.
|
|
5
37
|
|
|
@@ -135,39 +167,3 @@ When a change does not fit one layer clearly, say so in the MR instead of pickin
|
|
|
135
167
|
Mention the sync agent on the review thread to have it draft the scenario MR; if it cannot, open a labkit-spec issue and link it from the thread.
|
|
136
168
|
- Never edit Lua scripts, key construction, header names, metric names, or log field names to fix a Ruby-only bug.
|
|
137
169
|
If a fix seems to need that, it is a contract change.
|
|
138
|
-
=======
|
|
139
|
-
# Agent guidelines
|
|
140
|
-
|
|
141
|
-
This project uses [Conventional Commits](https://www.conventionalcommits.org/).
|
|
142
|
-
Commit messages drive automated [semantic releases](https://semantic-release.gitbook.io/)
|
|
143
|
-
(see `.releaserc.json`), so use `feat:`, `fix:`, etc. prefixes to ensure the
|
|
144
|
-
correct version bump and changelog entry.
|
|
145
|
-
|
|
146
|
-
## AI-authored GitLab comments
|
|
147
|
-
|
|
148
|
-
Wrap the full body of any comment, note, or reply you post to GitLab (issues,
|
|
149
|
-
merge requests, epics — via `glab`/`glab api`) in `<:robot:>` / `</:robot:>`
|
|
150
|
-
tags (GitLab emoji shortcode, renders as 🤖), so readers can tell at a glance
|
|
151
|
-
that it's AI-generated:
|
|
152
|
-
|
|
153
|
-
```
|
|
154
|
-
<:robot:>
|
|
155
|
-
|
|
156
|
-
...full comment body...
|
|
157
|
-
|
|
158
|
-
</:robot:>
|
|
159
|
-
```
|
|
160
|
-
|
|
161
|
-
This applies to newly posted comments. It does not apply to code, commit
|
|
162
|
-
messages, or MR/issue descriptions unless asked.
|
|
163
|
-
|
|
164
|
-
<!--
|
|
165
|
-
####################################################################
|
|
166
|
-
# 🌊✨ PLEASE ADD PROJECT SPECIFIC RULES BELOW THIS LINE ✨🌊 #
|
|
167
|
-
# The section above is managed by the common-template-copier #
|
|
168
|
-
# template and is updated automatically on `copier update`. #
|
|
169
|
-
# Add your project-specific instructions below this line so #
|
|
170
|
-
# they are preserved when the template is updated. #
|
|
171
|
-
####################################################################
|
|
172
|
-
-->
|
|
173
|
-
>>>>>>> after updating
|
data/CODEOWNERS
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
# CODEOWNERS is used to lookup assignees for
|
|
2
2
|
# Renovate Bot dependency change Merge Requests.
|
|
3
3
|
# https://docs.renovatebot.com/configuration-options/#assigneesfromcodeowners
|
|
4
|
-
* @
|
|
4
|
+
* @ayufan @hmerscher @e_forbes @M_Alvarez @mwoolf @sankalp_gl @hardikgala @nindurkar @ashs2
|
data/CONTRIBUTING.md
CHANGED
|
@@ -6,7 +6,7 @@ Except for the license granted herein to GitLab Inc. and recipients of software
|
|
|
6
6
|
distributed by GitLab Inc., you reserve all right, title, and interest in and to
|
|
7
7
|
your Contributions.
|
|
8
8
|
|
|
9
|
-
All contributions are subject to the Developer Certificate of Origin and license set out at [docs.gitlab.com/
|
|
9
|
+
All contributions are subject to the Developer Certificate of Origin and license set out at [docs.gitlab.com/legal/developer_certificate_of_origin](https://docs.gitlab.com/legal/developer_certificate_of_origin/).
|
|
10
10
|
|
|
11
11
|
_This notice should stay as the first item in the CONTRIBUTING.md file._
|
|
12
12
|
|
data/lib/labkit/fields.rb
CHANGED
|
@@ -16,6 +16,12 @@ module Labkit
|
|
|
16
16
|
# GitLab username.
|
|
17
17
|
GL_USER_NAME = "gl_user_name"
|
|
18
18
|
|
|
19
|
+
# Type of key used for authentication (e.g. "deploy_key", "key").
|
|
20
|
+
GL_KEY_TYPE = "gl_key_type"
|
|
21
|
+
|
|
22
|
+
# Numeric ID of the key used for authentication.
|
|
23
|
+
GL_KEY_ID = "gl_key_id"
|
|
24
|
+
|
|
19
25
|
# Duo Workflow definition identifier (e.g. "analytics_agent/v1"). Identifies
|
|
20
26
|
# which Duo Workflow agent originated a request, enabling per-agent
|
|
21
27
|
# filtering in Kibana and Grafana.
|
|
@@ -149,12 +155,19 @@ module Labkit
|
|
|
149
155
|
# connection field.
|
|
150
156
|
GRAPHQL_CONNECTION_NODES = "graphql_connection_nodes"
|
|
151
157
|
|
|
158
|
+
# Protocol or subsystem that produced an access log line (e.g. "http",
|
|
159
|
+
# "grpc"). Emitted by the LabKit access-logging middleware so a single
|
|
160
|
+
# service's HTTP and gRPC access logs can be told apart.
|
|
161
|
+
SYSTEM = "system"
|
|
162
|
+
|
|
152
163
|
# Maps each field constant to its logging field variant version.
|
|
153
164
|
# A version of 0 means the field is not yet assigned to any variant.
|
|
154
165
|
VARIANT_VERSION = {
|
|
155
166
|
CORRELATION_ID => 1,
|
|
156
167
|
GL_USER_ID => 1,
|
|
157
168
|
GL_USER_NAME => 1,
|
|
169
|
+
GL_KEY_TYPE => 0,
|
|
170
|
+
GL_KEY_ID => 0,
|
|
158
171
|
DUO_WORKFLOW_DEFINITION => 0,
|
|
159
172
|
DUO_WORKFLOW_ID => 0,
|
|
160
173
|
ERROR_TYPE => 0,
|
|
@@ -192,6 +205,7 @@ module Labkit
|
|
|
192
205
|
ADDITIONAL_DETAILS => 0,
|
|
193
206
|
GRAPHQL_COMPLEXITY_RAW => 0,
|
|
194
207
|
GRAPHQL_CONNECTION_NODES => 0,
|
|
208
|
+
SYSTEM => 0,
|
|
195
209
|
}.freeze
|
|
196
210
|
|
|
197
211
|
# Get the constant name for a field value
|
|
@@ -601,6 +601,56 @@ the request rate through a limiter — no exclusions or dedup needed. A single
|
|
|
601
601
|
`check` call emits **one** `checks_total` increment and as many
|
|
602
602
|
`rule_evaluations_total` increments as rules it evaluated (possibly zero).
|
|
603
603
|
|
|
604
|
+
## Block summary
|
|
605
|
+
|
|
606
|
+
`Labkit::RateLimit::BlockSummary` counts blocks per `(endpoint_id, rule)` in
|
|
607
|
+
one process and logs the counts every interval (300 seconds by default). The
|
|
608
|
+
number of lines grows with the number of distinct pairs the process blocked,
|
|
609
|
+
not with request rate, so an attack does not multiply log volume.
|
|
610
|
+
|
|
611
|
+
Each process tracks at most `max_pairs` pairs (10 by default) per interval.
|
|
612
|
+
Blocks for any further pair are counted on one `endpoint_id: "other"`,
|
|
613
|
+
`rate_limit_rule: "other"` line, with no `feature_category`, so a process
|
|
614
|
+
writes at most `max_pairs + 1` lines per flush and the tally's memory stays
|
|
615
|
+
bounded.
|
|
616
|
+
|
|
617
|
+
```ruby
|
|
618
|
+
BLOCK_SUMMARY = Labkit::RateLimit::BlockSummary.new(logger: logger)
|
|
619
|
+
|
|
620
|
+
# On every blocked request. feature_category is optional:
|
|
621
|
+
BLOCK_SUMMARY.add(endpoint_id: "GET /api/:version/projects", rule: "unauthenticated_api", feature_category: "source_code_management")
|
|
622
|
+
|
|
623
|
+
# The caller owns the thread. Start it in each worker process, after any fork:
|
|
624
|
+
thread = Thread.new { BLOCK_SUMMARY.call }
|
|
625
|
+
|
|
626
|
+
# On shutdown, flush what is left and let the thread return:
|
|
627
|
+
BLOCK_SUMMARY.stop
|
|
628
|
+
thread.join
|
|
629
|
+
```
|
|
630
|
+
|
|
631
|
+
Each flush writes one INFO line per pair and then empties the counts:
|
|
632
|
+
|
|
633
|
+
```json
|
|
634
|
+
{"message": "rate_limit_block_summary", "endpoint_id": "GET /api/:version/projects", "rate_limit_rule": "unauthenticated_api", "feature_category": "source_code_management", "block_count": 412}
|
|
635
|
+
```
|
|
636
|
+
|
|
637
|
+
`feature_category` is left out when the caller passed none. It does not split
|
|
638
|
+
a pair: the first non-nil one passed for a pair is the one logged. The first
|
|
639
|
+
flush waits a random part of the interval, so processes started together do
|
|
640
|
+
not all flush at once. A flush with nothing counted writes nothing. The caller
|
|
641
|
+
decides what `endpoint_id` means. LabKit does not resolve it. Counts in a
|
|
642
|
+
process that exits without `stop` are lost, up to one interval's worth. `stop`
|
|
643
|
+
only signals the thread in `call`, which runs the final flush, so it flushes
|
|
644
|
+
nothing if no thread is running `call`.
|
|
645
|
+
|
|
646
|
+
`interval` must be a positive number and `max_pairs` a positive Integer, or
|
|
647
|
+
`new` raises `ArgumentError`.
|
|
648
|
+
|
|
649
|
+
A logging failure during a flush is logged at WARN with
|
|
650
|
+
`message: "rate_limit_error"`, `error_type: "rate_limit_error"` and
|
|
651
|
+
`dropped_pairs`, the number of lines that flush lost. It never raises, so the
|
|
652
|
+
flush thread keeps running.
|
|
653
|
+
|
|
604
654
|
## Dev/test vs production guards
|
|
605
655
|
|
|
606
656
|
`Limiter.new` and `Rule.new` validate names and configuration. In
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Labkit
|
|
4
|
+
module RateLimit
|
|
5
|
+
# Counts blocks per (endpoint_id, rule) in this process and logs the counts
|
|
6
|
+
# every interval. A line per block grows with an attack, so lines here grow
|
|
7
|
+
# only with the number of distinct pairs this process blocked, and never past
|
|
8
|
+
# max_pairs + 1.
|
|
9
|
+
#
|
|
10
|
+
# One line per pair rather than one line holding a map: map keys become
|
|
11
|
+
# index fields, one per endpoint name. The fields are rate_limit_rule and
|
|
12
|
+
# block_count, not rule and count, so they cannot clash with another line's
|
|
13
|
+
# type for a common name in a shared index.
|
|
14
|
+
#
|
|
15
|
+
# The caller owns the thread: run #call on one, and #stop it on shutdown so
|
|
16
|
+
# the last counts are flushed.
|
|
17
|
+
class BlockSummary
|
|
18
|
+
MESSAGE = "rate_limit_block_summary"
|
|
19
|
+
OTHER = "other"
|
|
20
|
+
OTHER_KEY = [OTHER, OTHER].freeze
|
|
21
|
+
DEFAULT_INTERVAL_SECONDS = 300
|
|
22
|
+
DEFAULT_MAX_PAIRS = 10
|
|
23
|
+
|
|
24
|
+
def initialize(logger: nil, interval: DEFAULT_INTERVAL_SECONDS, max_pairs: DEFAULT_MAX_PAIRS)
|
|
25
|
+
# A zero interval would make #call spin without waiting.
|
|
26
|
+
raise ArgumentError, "interval must be a positive Numeric, got #{interval.inspect}" unless interval.is_a?(Numeric) && interval.positive?
|
|
27
|
+
|
|
28
|
+
raise ArgumentError, "max_pairs must be a positive Integer, got #{max_pairs.inspect}" unless max_pairs.is_a?(Integer) && max_pairs.positive?
|
|
29
|
+
|
|
30
|
+
@logger = logger || RateLimit.config.logger || Labkit::Logging::JsonLogger.new($stdout)
|
|
31
|
+
@interval = interval
|
|
32
|
+
@max_pairs = max_pairs
|
|
33
|
+
@counts = {}
|
|
34
|
+
@mutex = Mutex.new
|
|
35
|
+
@stop_signal = Thread::Queue.new
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Keyed on the pair alone, so feature_category never adds a line. The first
|
|
39
|
+
# non-nil one passed for a pair is kept. Capped here rather than at flush,
|
|
40
|
+
# so the hash cannot grow during an attack.
|
|
41
|
+
def add(endpoint_id:, rule:, feature_category: nil)
|
|
42
|
+
key = [endpoint_id, rule]
|
|
43
|
+
|
|
44
|
+
@mutex.synchronize do
|
|
45
|
+
key = OTHER_KEY if !@counts.key?(key) && pairs_full?
|
|
46
|
+
entry = (@counts[key] ||= [0, nil])
|
|
47
|
+
entry[0] += 1
|
|
48
|
+
entry[1] ||= feature_category unless key == OTHER_KEY
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Waits on a queue rather than sleeping, so a stop that lands mid-flush is
|
|
53
|
+
# still seen and the final flush is never skipped. The first wait is
|
|
54
|
+
# random, so workers booted together do not all flush at once.
|
|
55
|
+
def call
|
|
56
|
+
timeout = rand * @interval
|
|
57
|
+
|
|
58
|
+
loop do
|
|
59
|
+
stopping = @stop_signal.pop(timeout: timeout)
|
|
60
|
+
flush
|
|
61
|
+
break if stopping
|
|
62
|
+
|
|
63
|
+
timeout = @interval
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Only signals #call, which runs the final flush. Without a thread in #call,
|
|
68
|
+
# nothing is flushed.
|
|
69
|
+
def stop
|
|
70
|
+
@stop_signal << true
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def flush
|
|
74
|
+
drained = {}
|
|
75
|
+
logged = 0
|
|
76
|
+
|
|
77
|
+
drained = @mutex.synchronize do
|
|
78
|
+
counts = @counts
|
|
79
|
+
@counts = {}
|
|
80
|
+
counts
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
drained.each do |(endpoint_id, rule), (count, feature_category)|
|
|
84
|
+
@logger.info(line(endpoint_id, rule, feature_category, count))
|
|
85
|
+
logged += 1
|
|
86
|
+
end
|
|
87
|
+
rescue StandardError => e
|
|
88
|
+
log_failure(e, drained.size - logged)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def pairs_full?
|
|
94
|
+
@counts.size - (@counts.key?(OTHER_KEY) ? 1 : 0) >= @max_pairs
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def line(endpoint_id, rule, feature_category, count)
|
|
98
|
+
payload = { message: MESSAGE, Labkit::Fields::ENDPOINT_ID => endpoint_id, rate_limit_rule: rule, block_count: count }
|
|
99
|
+
payload[:feature_category] = feature_category if feature_category
|
|
100
|
+
payload
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Never raises: a raise here would kill the caller's flush thread.
|
|
104
|
+
# The counts are already drained, so dropped_pairs says how many lines
|
|
105
|
+
# this flush lost.
|
|
106
|
+
def log_failure(error, dropped_pairs)
|
|
107
|
+
@logger.warn(
|
|
108
|
+
message: "rate_limit_error",
|
|
109
|
+
Labkit::Fields::ERROR_TYPE => "rate_limit_error",
|
|
110
|
+
Labkit::Fields::CLASS_NAME => error.class.to_s,
|
|
111
|
+
Labkit::Fields::ERROR_MESSAGE => error.message,
|
|
112
|
+
dropped_pairs: dropped_pairs
|
|
113
|
+
)
|
|
114
|
+
rescue StandardError
|
|
115
|
+
nil
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
data/lib/labkit/rate_limit.rb
CHANGED
|
@@ -25,6 +25,7 @@ module Labkit
|
|
|
25
25
|
autoload :Evaluator, "labkit/rate_limit/evaluator"
|
|
26
26
|
autoload :Limiter, "labkit/rate_limit/limiter"
|
|
27
27
|
autoload :Metrics, "labkit/rate_limit/metrics"
|
|
28
|
+
autoload :BlockSummary, "labkit/rate_limit/block_summary"
|
|
28
29
|
|
|
29
30
|
class << self
|
|
30
31
|
def configure
|
data/scripts/prepare-dev-env.sh
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab-labkit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.
|
|
4
|
+
version: 5.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Newdigate
|
|
@@ -651,6 +651,7 @@ files:
|
|
|
651
651
|
- lib/labkit/net_http_publisher.rb
|
|
652
652
|
- lib/labkit/rate_limit.rb
|
|
653
653
|
- lib/labkit/rate_limit/README.md
|
|
654
|
+
- lib/labkit/rate_limit/block_summary.rb
|
|
654
655
|
- lib/labkit/rate_limit/configuration.rb
|
|
655
656
|
- lib/labkit/rate_limit/evaluator.rb
|
|
656
657
|
- lib/labkit/rate_limit/identifier.rb
|