@jutge.org/toolkit 4.4.9 → 4.4.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +1 -0
  2. package/assets/prompts/funcs/creators/create-solution.tpl.txt +23 -0
  3. package/assets/prompts/funcs/creators/create-statement.tpl.txt +28 -0
  4. package/assets/prompts/funcs/creators/private-test-cases.tpl.txt +12 -0
  5. package/assets/prompts/funcs/creators/sample-test-cases.tpl.txt +16 -0
  6. package/assets/prompts/funcs/creators/sample.dt.tpl.txt +15 -0
  7. package/assets/prompts/funcs/examples/statement.tex +23 -0
  8. package/assets/prompts/funcs/proglangs/py.md +9 -0
  9. package/assets/prompts/io/examples/statement-coda.tex +7 -0
  10. package/assets/python/pyexec.py +9 -2
  11. package/dist/index.js +707 -467
  12. package/docs/getting-started-guide.md +1 -1
  13. package/docs/login.md +0 -1
  14. package/docs/problem-anatomy.md +1 -1
  15. package/docs/quiz-anatomy.md +367 -0
  16. package/package.json +14 -11
  17. package/toolkit/ask.ts +1 -1
  18. package/toolkit/dummies.ts +5 -4
  19. package/toolkit/generate.ts +83 -14
  20. package/toolkit/lint.ts +3 -4
  21. package/toolkit/make.ts +20 -23
  22. package/toolkit/quiz.ts +22 -14
  23. package/toolkit/share.ts +5 -4
  24. package/toolkit/submit.ts +4 -1
  25. /package/assets/prompts/{examples → funcs/examples}/statement-coda.tex +0 -0
  26. /package/assets/prompts/{creators → io/creators}/create-solution.tpl.txt +0 -0
  27. /package/assets/prompts/{creators → io/creators}/create-statement.tpl.txt +0 -0
  28. /package/assets/prompts/{creators → io/creators}/create-translation.tpl.txt +0 -0
  29. /package/assets/prompts/{creators → io/creators}/private-test-cases.txt +0 -0
  30. /package/assets/prompts/{creators → io/creators}/sample-test-cases.txt +0 -0
  31. /package/assets/prompts/{examples → io/examples}/statement.tex +0 -0
  32. /package/assets/prompts/{generators → io/generators}/efficiency.md +0 -0
  33. /package/assets/prompts/{generators → io/generators}/hard.md +0 -0
  34. /package/assets/prompts/{generators → io/generators}/random.md +0 -0
  35. /package/assets/prompts/{proglangs → io/proglangs}/cc.md +0 -0
  36. /package/assets/prompts/{proglangs → io/proglangs}/py.md +0 -0
@@ -1,6 +1,6 @@
1
1
  # Jutge Toolkit - Getting Started Guide
2
2
 
3
- Welcome to the New Jutge Toolkit! This guide will help you start using the toolkit to create, manage, and upload programming problems to Jutge.org. Information about problem formats will be provided in a separate document.
3
+ Welcome to the New Jutge Toolkit! This guide will help you start using the toolkit to create, manage, and upload programming problems to Jutge.org. For general problem structure and file formats, see [Problem anatomy](problem-anatomy.md); for quiz problems, see [Quiz anatomy](quiz-anatomy.md).
4
4
 
5
5
  ## What is Jutge Toolkit?
6
6
 
package/docs/login.md CHANGED
@@ -1 +0,0 @@
1
-
@@ -34,7 +34,7 @@ There are four types of problems in Jutge.org:
34
34
 
35
35
  - **Games**: Problems where users must implement an AI to play a game against other users or predefined strategies. These problems often involve turn-based gameplay and require users to implement specific functions to decide their moves. These are not covered in this document.
36
36
 
37
- - **Quizzes**: Problems where users must answer multiple-choice questions. These are not covered in this document.
37
+ - **Quizzes**: Problems where users must answer multiple-choice, fill-in, ordering, matching, or open-ended questions. See [Quiz anatomy](quiz-anatomy.md) for details.
38
38
 
39
39
  ## Problem structure
40
40
 
@@ -0,0 +1,367 @@
1
+ # Quiz anatomy
2
+
3
+ This document describes the anatomy of a **quiz problem** in [Jutge.org](https://jutge.org/). It explains the terminology used, the structure of a quiz, and the purpose of each file. Quiz problems are a type of problem where users answer multiple-choice, fill-in, ordering, matching, or open-ended questions instead of submitting a program. Quiz questions and answers may be randomized for each run. For the general structure of a problem folder (e.g. `.pbm` extension, `problem.yml`, statement files), see [Problem anatomy](problem-anatomy.md).
4
+
5
+ ## Terminology
6
+
7
+ A **quiz** is a problem whose handler is set to `quiz`. It is made of a **quiz root** and a list of **questions**. The quiz root is defined in `quiz.yml` and holds the quiz title, statement, whether questions are shuffled, and the list of questions with their scores. Each **question** is defined in a separate YAML file (e.g. `single-choice.yml`) and has a **type**: SingleChoice, MultipleChoice, FillIn, Ordering, Matching, or OpenQuestion.
8
+
9
+ Quiz content can be **localized**: the same quiz can have different `quiz.yml` and question files per language (e.g. under `en/` and `ca/`). The toolkit runs or lints the quiz for a given directory, so you typically run it from a language-specific subdirectory. Each language should live inside its own folder.
10
+
11
+ **Variable substitution** allows question text and options to depend on values generated at run time. If a question file is named `example.yml`, the toolkit looks for `example.py` in the same directory. When present, it runs the Python script with a random **seed** and collects the script’s global variables. Those variables can be referenced in the question YAML with `$name` or `${name}`. This makes it possible to have different numbers, strings, or options for each run while keeping the same correct answer logic (e.g. “What is $a + $b?” with `a` and `b` random).
12
+
13
+ **Scoring**: Each question has a **score** between 0–100, and the total of all question scores listed in `quiz.yml` must add up to 100. Users earn points for each question based on question type and that question’s **partial_answer** setting. The **partial_answer** option is set per question in the question YAML (not in `quiz.yml`):
14
+
15
+ - If `partial_answer` is set to `false` (default), users get full points for that question only when their answer is completely correct; any mistake gives zero points for that question.
16
+
17
+ - If `partial_answer` is set to `true`, users can receive partial points for that question when the answer is partially correct (e.g. proportional to how many parts are right), and the response may still be marked as "correct" if at least one part is right.
18
+
19
+ ## Quiz structure
20
+
21
+ A quiz lives inside a problem folder (e.g. a `.pbm`). Each language should live inside its own folder (e.g. `en/`, `ca/`) .
22
+
23
+ Example with multiple languages:
24
+
25
+ ```
26
+ problem_folder.pbm
27
+ ├── en
28
+ │ ├── handler.yml
29
+ │ ├── quiz.yml
30
+ │ ├── some-question.yml
31
+ │ ├── some-question.py
32
+ │ └── ...
33
+ ├── ca
34
+ │ ├── handler.yml
35
+ │ ├── quiz.yml
36
+ │ ├── some-question.yml
37
+ │ ├── some-question.py
38
+ │ └── ...
39
+ └── problem.yml
40
+ ```
41
+
42
+ You run or lint the quiz from the directory that contains `quiz.yml` (e.g. `jtk quiz run -d en` or `cd en && jtk quiz run`).
43
+
44
+ `yml` files are YAML (YAML Ain't Markup Language) files. YAML is a human-readable data-serialization language; see [YAML documentation](https://yaml.org/) for more details. Also, see [YAML multiline info](https://yaml-multiline.info/) for more details on how to write multiline strings in YAML.
45
+
46
+ Many items are written in Markdown. See [Markdown documentation](https://www.markdownguide.org/) for more details. In addition, you can use a small subset of LaTeX for mathematical expressions but these have to be enclosed between `·` signs, not standard `$` signs.
47
+
48
+ ## The `quiz.yml` file
49
+
50
+ The file `quiz.yml` defines the quiz root.
51
+
52
+ - **title**: Title of the quiz.
53
+ - **statement**: Short description or instructions shown for the quiz (Markdown).
54
+ - **questions**: List of question entries. Each entry has:
55
+ - **title**: Title of the question (e.g. for display in a table of contents).
56
+ - **file**: Base name of the question file, without the `.yml` extension (e.g. `question` for `question.yml`).
57
+ - **score**: Integer from 0 to 100. The sum of all `score` values in the list must be 100.
58
+ - **shuffle** (optional): Whether to shuffle the order of questions when running the quiz. Defaults to `false`.
59
+
60
+ ### Example
61
+
62
+ ```yaml
63
+ title: Demo quiz
64
+
65
+ statement: This quiz showcases the possibilities of the quiz problems at Jutge.org.
66
+
67
+ shuffle: true
68
+
69
+ questions:
70
+ - title: Single choice question
71
+ file: single-choice
72
+ score: 10
73
+
74
+ - title: Multiple choice question
75
+ file: multiple-choice
76
+ score: 10
77
+
78
+ - title: Fill in question
79
+ file: fill-in-1
80
+ score: 20
81
+
82
+ - title: Ordering question
83
+ file: ordering
84
+ score: 10
85
+
86
+ - title: Matchings question
87
+ file: matchings
88
+ score: 20
89
+ ```
90
+
91
+ ## Question types
92
+
93
+ Each question is stored in a YAML file whose name matches the `file` field in `quiz.yml` (e.g. `question.yml`). The file must contain a **type** field that identifies the kind of question. All question types support an optional **hide_score** (default `false`) and an optional **partial_answer** (default `false`); see [Scoring](#terminology) above. Variable substitution applies to text fields and options when a corresponding `.py` file exists.
94
+
95
+ ### SingleChoice
96
+
97
+ One correct option among several. Exactly one choice must have `correct: true`. Choices can be shuffled (optional **shuffle**, default `true`). Each choice can have an optional **hint**. Duplicate choice text is not allowed.
98
+
99
+ - **text**: Question text (supports `$var` and `${var}`).
100
+ - **choices**: List of `{ text, correct?, hint? }`. One and only one choice must have `correct: true`.
101
+ - **shuffle** (optional): Whether to shuffle choices. Defaults to `true`.
102
+ - **partial_answer** (optional): Whether to award partial credit for this question. Defaults to `false`.
103
+
104
+ Example:
105
+
106
+ ```yaml
107
+ type: SingleChoice
108
+
109
+ text: 'What is the result of the evaluation of `$a + $b`?'
110
+
111
+ choices:
112
+ - text: '`$s1`'
113
+ correct: true
114
+ hint: 'You did well'
115
+ - text: '`$s2`'
116
+ hint: 'Sorry...'
117
+ - text: '`$s3`'
118
+ ```
119
+
120
+ Variables `a`, `b`, `s1`, `s2`, `s3` would be produced by a `single-choice.py` script in the same directory.
121
+
122
+ ### MultipleChoice
123
+
124
+ Zero or more correct options. Multiple choices can have `correct: true`. Choices can be shuffled (optional **shuffle**, default `true`).
125
+
126
+ - **text**: Question text (supports variables).
127
+ - **choices**: List of `{ text, correct?, hint? }`.
128
+ - **shuffle** (optional): Whether to shuffle choices. Defaults to `true`.
129
+ - **partial_answer** (optional): Whether to award partial credit for this question. Defaults to `false`.
130
+
131
+ Example:
132
+
133
+ ```yaml
134
+ type: MultipleChoice
135
+
136
+ text: 'Which of the following expressions are `true`?'
137
+
138
+ choices:
139
+ - text: '`$s == $a + $b`'
140
+ correct: true
141
+ - text: '`$s != $a + $b`'
142
+ hint: 'Sorry...'
143
+ - text: '`$a + $b >= $s`'
144
+ correct: true
145
+ ```
146
+
147
+ ### FillIn
148
+
149
+ One or more blanks in a text or code block. Each blank is identified by a placeholder (e.g. `S1`, `XXXX`) and has a correct answer and optional options (dropdown). If **options** are given, the correct answer must be one of them.
150
+
151
+ - **text**: Question or instructions (supports variables).
152
+ - **context**: Text containing placeholders (e.g. `S1`, `S2`, `XXXX`). Placeholders are the keys in **items**.
153
+ - **items**: Map from placeholder name to an item object:
154
+ - **correct**: Correct answer (string).
155
+ - **options** (optional): List of strings; if present, the blank is shown as a dropdown and **correct** must be in this list.
156
+ - **maxlength** (optional): Max length for the answer. Defaults to 100.
157
+ - **placeholder** (optional): Placeholder text in the input (e.g. `"?"`).
158
+ - **ignorecase** (optional): Whether to ignore case when checking. Defaults to `true`.
159
+ - **trim** (optional): Whether to trim spaces. Defaults to `true`.
160
+ - **partial_answer** (optional): Whether this blank contributes to partial credit for the question. Defaults to `false`.
161
+ - **partial_answer** (optional, at question level): Whether to award partial credit for this question. Defaults to `false`.
162
+
163
+ Example with dropdowns:
164
+
165
+ ```yaml
166
+ type: FillIn
167
+
168
+ text: 'Fill in the blanks.'
169
+
170
+ context: |
171
+ A/An S1 is a self-contained step-by-step set of operations...
172
+ A/An S2 is a collection of instructions...
173
+
174
+ items:
175
+ S1:
176
+ correct: algorithm
177
+ options:
178
+ - program
179
+ - algorithm
180
+ - pseudocode
181
+ S2:
182
+ correct: program
183
+ options:
184
+ - program
185
+ - algorithm
186
+ ```
187
+
188
+ Example with free-text blanks (e.g. numeric):
189
+
190
+ ````yaml
191
+ type: FillIn
192
+
193
+ text: |
194
+ Fill in the blanks of the given program.
195
+
196
+ context: |
197
+ ```python
198
+ def sum (L):
199
+ s = XXXX
200
+ for x in L:
201
+ s = s YYYY x
202
+ return s
203
+ ```
204
+
205
+ items:
206
+ XXXX:
207
+ maxlength: 5
208
+ correct: 0
209
+ YYYY:
210
+ correct: '+'
211
+ options:
212
+ - '+'
213
+ - '-'
214
+ - '*'
215
+ - '/'
216
+ ````
217
+
218
+ ### Ordering
219
+
220
+ User must order a list of items (e.g. chronological order). Items can be shown in shuffled order (optional **shuffle**, default `true`).
221
+
222
+ - **text**: Question text (supports variables).
223
+ - **label**: Label for the list (e.g. “Programming language”).
224
+ - **items**: List of strings in the **correct** order.
225
+ - **shuffle** (optional): Whether to show items in random order. Defaults to `true`.
226
+ - **partial_answer** (optional): Whether to award partial credit for this question. Defaults to `false`.
227
+
228
+ Example:
229
+
230
+ ```yaml
231
+ type: Ordering
232
+
233
+ text: Drag and drop the item to order the programming languages by date of appearance (older on top).
234
+
235
+ label: Programming language
236
+
237
+ items:
238
+ - Fortran
239
+ - C
240
+ - C++
241
+ - Python
242
+ - Java
243
+ - Julia
244
+ ```
245
+
246
+ ### Matching
247
+
248
+ Two columns: user matches each left item with one right item. Left and right lists can be shuffled (optional **shuffle**, default `true`).
249
+
250
+ - **text**: Question text (supports variables).
251
+ - **labels**: Two strings, e.g. `["Countries", "Capitals"]`.
252
+ - **left**: List of strings (e.g. countries).
253
+ - **right**: List of strings (e.g. capitals), in the same order as **left** (left[i] matches right[i]).
254
+ - **shuffle** (optional): Whether to shuffle left and right columns. Defaults to `true`.
255
+ - **partial_answer** (optional): Whether to award partial credit for this question. Defaults to `false`.
256
+
257
+ Example:
258
+
259
+ ```yaml
260
+ type: Matching
261
+
262
+ text: Match the countries with their capitals.
263
+
264
+ labels:
265
+ - Countries
266
+ - Capitals
267
+
268
+ left:
269
+ - France
270
+ - Germany
271
+ - Spain
272
+ - Italy
273
+ - Andorra
274
+
275
+ right:
276
+ - Paris
277
+ - Berlin
278
+ - Madrid
279
+ - Rome
280
+ - Andorra la Vella
281
+ ```
282
+
283
+ ### OpenQuestion
284
+
285
+ Free-text answer with no automatic correction. Useful for open-ended or reflective answers.
286
+
287
+ - **text**: Question text (supports variables).
288
+ - **placeholder** (optional): Placeholder for the text area. Defaults to `""`. Supports variables.
289
+ - **partial_answer** (optional): Whether to award partial credit for this question. Defaults to `false`.
290
+
291
+ Example:
292
+
293
+ ```yaml
294
+ type: OpenQuestion
295
+
296
+ text: Talk about yourself.
297
+
298
+ placeholder: 'My name is **$name** and I want to pass this course.'
299
+ ```
300
+
301
+ The variable `name` can be set by an optional `open-ended.py` (or the same base name as the question file) in the same directory.
302
+
303
+ ## Variable substitution (`.py` files)
304
+
305
+ If a question file is named `example.yml`, the toolkit looks for `example.py` in the same directory. When present:
306
+
307
+ 1. The Python script is run with a given **seed** (passed as an argument by the toolkit) so that the run is reproducible.
308
+ 2. The script’s global variables (that are JSON-serializable and whose names do not start with `__`) are collected.
309
+ 3. In the question YAML, any string field that supports substitution can use `$name` or `${name}` to be replaced by the value of `name`.
310
+
311
+ This allows different runs to show different numbers or options while keeping the same correct answer (e.g. “What is $a + $b?” with `a` and `b` random and one choice `$s1 = a + b` marked correct).
312
+
313
+ Example `question.py`:
314
+
315
+ ```python
316
+ import random
317
+
318
+ a = random.randint(1, 10)
319
+ b = random.randint(1, 10)
320
+
321
+ s1 = a + b
322
+ s2 = a + b - 1
323
+ s3 = a + b + 1
324
+ s4 = a - b
325
+ s5 = a * b
326
+ ```
327
+
328
+ Used with a SingleChoice question where the correct answer is `$s1`, so each run has different numbers but the same structure.
329
+
330
+ ## The `handler.yml` file
331
+
332
+ For a quiz problem, the problem’s `handler.yml` must set the handler to `quiz`:
333
+
334
+ ```yaml
335
+ handler: quiz
336
+ ```
337
+
338
+ Other handler options (e.g. `std`, `graphic`) are for non-quiz problems. See [Problem anatomy — handler.yml](problem-anatomy.md#the-handleryml-file) for the full list of handler and option descriptions.
339
+
340
+ ## Running and linting quizzes
341
+
342
+ From the toolkit CLI:
343
+
344
+ - **Lint** a quiz (validate `quiz.yml` and all referenced question YAML files):
345
+
346
+ ```bash
347
+ jtk quiz lint -d <directory>
348
+ ```
349
+
350
+ Use the directory that contains `quiz.yml` (e.g. the `en` subdirectory).
351
+
352
+ - **Run** a quiz (build questions, apply variables, output JSON or YAML):
353
+ ```bash
354
+ jtk quiz run -d <directory> [-s <seed>] [-f json|yaml]
355
+ ```
356
+ If no seed is provided, a random one is used. Running the quiz applies variable substitution and, if `shuffle` is true, shuffles question order (and, per question, choices or ordering/matching items when their `shuffle` is true).
357
+
358
+ ## Quick checklist
359
+
360
+ - Use a problem folder with `handler: quiz` in `handler.yml`.
361
+ - Place `quiz.yml` in the root (single-language) or in a per-language subdirectory (e.g. `en/`, `ca/`).
362
+ - Ensure every `file` in `quiz.yml` has a corresponding `file.yml` in the same directory.
363
+ - Ensure question scores in `quiz.yml` sum to 100.
364
+ - For SingleChoice, set exactly one choice with `correct: true`.
365
+ - For FillIn with **options**, ensure **correct** is in the **options** list.
366
+ - For Matching, ensure **left** and **right** have the same length and are in matching order.
367
+ - Use variable names in `$name` / `${name}` that are produced by the optional `file.py` script; the script is run with the toolkit’s seed for reproducibility.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@jutge.org/toolkit",
3
3
  "description": "Toolkit to prepare problems for Jutge.org",
4
- "version": "4.4.9",
4
+ "version": "4.4.17",
5
5
  "homepage": "https://jutge.org",
6
6
  "author": {
7
7
  "name": "Jutge.org",
@@ -61,31 +61,33 @@
61
61
  ],
62
62
  "dependencies": {
63
63
  "@commander-js/extra-typings": "^14.0.0",
64
- "@dicebear/collection": "^9.3.1",
65
- "@dicebear/core": "^9.3.1",
66
- "@eslint/js": "^9.39.2",
67
- "@inquirer/prompts": "^8.2.0",
64
+ "@dicebear/collection": "^9.3.2",
65
+ "@dicebear/core": "^9.3.2",
66
+ "@eslint/js": "^10.0.1",
67
+ "@inquirer/prompts": "^8.3.0",
68
68
  "adm-zip": "^0.5.16",
69
69
  "archiver": "^7.0.1",
70
70
  "boxen": "^8.0.1",
71
- "bun-types": "^1.3.7",
71
+ "bun-types": "^1.3.9",
72
72
  "chalk": "^5.6.2",
73
73
  "chokidar": "^5.0.0",
74
74
  "cli-highlight": "^2.1.11",
75
- "commander": "^14.0.2",
75
+ "commander": "^14.0.3",
76
76
  "dayjs": "^1.11.19",
77
77
  "env-paths": "^4.0.0",
78
- "eslint": "^9.39.2",
78
+ "eslint": "^10.0.2",
79
79
  "execa": "^9.6.1",
80
80
  "handlebars": "^4.7.8",
81
81
  "image-size": "^2.0.2",
82
82
  "inquirer-checkbox-plus-plus": "^1.1.1",
83
+ "marked": "^17.0.3",
84
+ "marked-terminal": "^7.3.0",
83
85
  "nanoid": "^5.1.6",
84
86
  "open": "^11.0.0",
85
87
  "pretty-bytes": "^7.1.0",
86
88
  "pretty-ms": "^9.3.0",
87
89
  "radash": "^12.1.1",
88
- "semver": "^7.7.3",
90
+ "semver": "^7.7.4",
89
91
  "sharp": "^0.34.5",
90
92
  "terminal-link": "^5.0.0",
91
93
  "tree-node-cli": "^1.6.0",
@@ -98,10 +100,11 @@
98
100
  "@types/adm-zip": "^0.5.0",
99
101
  "@types/archiver": "^7.0.0",
100
102
  "@types/image-size": "^0.8.0",
101
- "@types/node": "^25.1.0",
103
+ "@types/marked-terminal": "^6.1.1",
104
+ "@types/node": "^25.3.0",
102
105
  "@types/semver": "^7.7.1",
103
106
  "prettier": "^3.8.1",
104
- "typescript-eslint": "^8.54.0"
107
+ "typescript-eslint": "^8.56.1"
105
108
  },
106
109
  "peerDependencies": {
107
110
  "typescript": "^5.9.3"
package/toolkit/ask.ts CHANGED
@@ -21,7 +21,7 @@ export const askCmd = new Command('ask')
21
21
  const docs = await loadDocumentation() // Load your markdown files
22
22
  const fullPrompt = `${docs}\n\nUser question: ${question}`
23
23
 
24
- const answer = await complete(jutge, model, "ask", systemPrompt, fullPrompt)
24
+ const answer = await complete(jutge, model, 'ask', systemPrompt, fullPrompt)
25
25
  await tui.markdown(answer)
26
26
  tui.warning(
27
27
  `This answer generated by JutgeAI using ${model} is not authoritative but we hope it will help you.`,
@@ -50,7 +50,6 @@ async function chooseCommand(
50
50
  return chooseCommand(program, path.slice(0, -1))
51
51
  }
52
52
 
53
-
54
53
  const newPath = [...path, chosen]
55
54
  const nextCmd = resolveCommand(program, newPath)!
56
55
  const nextSubs = getVisibleSubcommands(nextCmd)
@@ -91,10 +90,12 @@ async function promptForArgument(arg: Argument, existing: string[]): Promise<str
91
90
 
92
91
  const defaultStr =
93
92
  defaultVal != null
94
- ? (Array.isArray(defaultVal) ? defaultVal.join(' ') : String(defaultVal))
93
+ ? Array.isArray(defaultVal)
94
+ ? defaultVal.join(' ')
95
+ : String(defaultVal)
95
96
  : required
96
- ? undefined
97
- : ''
97
+ ? undefined
98
+ : ''
98
99
 
99
100
  const raw = await input({
100
101
  message,
@@ -1,8 +1,10 @@
1
- import { Argument, Command } from '@commander-js/extra-typings'
1
+ import { Argument, Command, Option } from '@commander-js/extra-typings'
2
+ import { exists } from 'fs/promises'
2
3
  import { join } from 'path'
3
4
  import sharp from 'sharp'
4
- import { createProblemWithJutgeAI } from '../lib/create-with-jutgeai'
5
5
  import { complete } from '../lib/aiclient'
6
+ import { createFuncsProblem } from '../lib/create-funcs'
7
+ import { createIOProblem } from '../lib/create-io'
6
8
  import { languageKeys, languageNames, proglangKeys, proglangNames } from '../lib/data'
7
9
  import {
8
10
  addAlternativeSolution,
@@ -11,11 +13,11 @@ import {
11
13
  generateStatementFromSolution,
12
14
  generateTestCasesGenerator,
13
15
  } from '../lib/generate'
16
+ import { getLoggedInJutgeClient } from '../lib/login'
14
17
  import { newProblem } from '../lib/problem'
15
18
  import { settings } from '../lib/settings'
16
19
  import tui from '../lib/tui'
17
20
  import { writeText } from '../lib/utils'
18
- import { getLoggedInJutgeClient } from '../lib/login'
19
21
 
20
22
  export const generateCmd = new Command('generate')
21
23
  .description('Generate problem elements using JutgeAI')
@@ -28,16 +30,76 @@ generateCmd
28
30
  .command('problem')
29
31
  .description('Generate a problem with JutgeAI')
30
32
 
33
+ .summary(
34
+ `Generate a problem with JutgeAI
35
+
36
+ Use this command to generate a problem with JutgeAI from a specification.
37
+
38
+ There are currently two types of problems that can be generated:
39
+
40
+ - io: The problem consists of reading input from standard input and writing output to standard output.
41
+
42
+ Current implementation supports C, C++, Python, Haskell, Java, Rust, R and Clojure programming languages.
43
+
44
+ The following items will be generated:
45
+ - problem statement in original language
46
+ - sample test cases
47
+ - private test cases
48
+ - golden solution
49
+ - translations of the problem statement into other languages
50
+ - alternative solutions in other programming languages
51
+ - test cases generators
52
+ - a README.md file describing the problem and LLM usage
53
+
54
+ - funcs: The problem consists of implementing one or more functions.
55
+
56
+ Current implementation supports Python, Haskell and Clojure programming languages (through RunPython, RunHaskell and RunClojure compilers).
57
+
58
+ The following items will be generated:
59
+ - problem statement in original language
60
+ - translations of the problem statement into other languages
61
+ - generate sample.dt for Python
62
+ - sample test cases
63
+ - private test cases for each function
64
+ - golden solution
65
+ - alternative solutions in other programming languages
66
+ - scores.yml file with the scores for each function (if there is more than one function)
67
+ - a README.md file describing the problem and LLM usage
68
+
69
+ Problem generation needs a problem specification:
70
+ - If --input is provided, the system will read the given input specification file.
71
+ - If --output is provided, the system will write the problem specification to the given output specification file.
72
+ - The system will ask interactively for the problem specification (using the values in the --input specification file if provided as defaults)
73
+ - unless the --do-not-ask flag is given.
74
+
75
+ Treat the generated problem as a starting draft. You should edit the problem directory manually after the generation.
76
+ `,
77
+ )
78
+
79
+ .addOption(new Option('-k, --kind <kind>', 'problem kind').default('io').choices(['io', 'funcs']))
31
80
  .option('-d, --directory <path>', 'output directory', 'new-problem.pbm')
32
81
  .option('-i, --input <path>', 'input specification file')
33
82
  .option('-o, --output <path>', 'output specification file')
34
83
  .option('-n, --do-not-ask', 'do not ask interactively if --input given', false)
35
84
  .option('-m, --model <model>', 'AI model to use', settings.defaultModel)
36
85
 
37
- .action(async ({ input, output, directory, model, doNotAsk }) => {
86
+ .action(async ({ input, output, directory, model, doNotAsk, kind }) => {
38
87
  const jutge = await getLoggedInJutgeClient()
39
- await tui.section('Generating problem with JutgeAI', async () => {
40
- await createProblemWithJutgeAI(jutge, model, directory, input, output, doNotAsk)
88
+ await tui.section(`Generating ${kind} problem with JutgeAI`, async () => {
89
+ if (await exists(directory)) {
90
+ throw new Error(`Directory ${directory} already exists`)
91
+ }
92
+ if (!directory.endsWith('.pbm')) {
93
+ throw new Error('The output directory must end with .pbm')
94
+ }
95
+
96
+ if (kind === 'io') {
97
+ await createIOProblem(jutge, model, directory, input, output, doNotAsk)
98
+ } else if (kind === 'funcs') {
99
+ await createFuncsProblem(jutge, model, directory, input, output, doNotAsk)
100
+ } else {
101
+ throw new Error(`Invalid problem kind: ${kind as string}`)
102
+ }
41
103
  })
42
104
  })
43
105
 
@@ -52,8 +114,8 @@ The original statement will be used as the source text for translation.
52
114
 
53
115
  Provide one or more target language from the following list:
54
116
  ${Object.entries(languageNames)
55
- .map(([key, name]) => ` - ${key}: ${name}`)
56
- .join('\n')}
117
+ .map(([key, name]) => ` - ${key}: ${name}`)
118
+ .join('\n')}
57
119
 
58
120
  The added translations will be saved in the problem directory overwrite possible existing files.`,
59
121
  )
@@ -94,7 +156,14 @@ The result is written to statement.<lang>.tex in the problem directory.`,
94
156
  .action(async (proglang, language, prompt, { directory, model }) => {
95
157
  const jutge = await getLoggedInJutgeClient()
96
158
  const problem = await newProblem(directory)
97
- await generateStatementFromSolution(jutge, model, problem, proglang, language, (prompt ?? '').trim() || undefined)
159
+ await generateStatementFromSolution(
160
+ jutge,
161
+ model,
162
+ problem,
163
+ proglang,
164
+ language,
165
+ (prompt ?? '').trim() || undefined,
166
+ )
98
167
  })
99
168
 
100
169
  generateCmd
@@ -108,8 +177,8 @@ The golden solution will be used as a reference for generating the alternatives.
108
177
 
109
178
  Provide one or more target programming languages from the following list:
110
179
  ${Object.entries(languageNames)
111
- .map(([key, name]) => ` - ${key}: ${name}`)
112
- .join('\n')}
180
+ .map(([key, name]) => ` - ${key}: ${name}`)
181
+ .join('\n')}
113
182
 
114
183
  The added solutions will be saved in the problem directory overwrite possible existing files.`,
115
184
  )
@@ -143,8 +212,8 @@ The main file for the golden solution will be used as a reference for generating
143
212
 
144
213
  Provide one or more target programming languages from the following list:
145
214
  ${Object.entries(languageNames)
146
- .map(([key, name]) => ` - ${key}: ${name}`)
147
- .join('\n')}
215
+ .map(([key, name]) => ` - ${key}: ${name}`)
216
+ .join('\n')}
148
217
 
149
218
  The added main files will be saved in the problem directory overwrite possible existing files.`,
150
219
  )
@@ -211,7 +280,7 @@ The new image will be saved as award.png in the problem directory, overriding an
211
280
  .action(async (prompt, { directory, model }) => {
212
281
  const jutge = await getLoggedInJutgeClient()
213
282
  const output = join(directory, 'award.png')
214
- const problem = await newProblem(directory)
283
+ await newProblem(directory)
215
284
  let imagePrompt = prompt.trim()
216
285
  if (imagePrompt === '') {
217
286
  imagePrompt = 'A colorful award on a white background'
package/toolkit/lint.ts CHANGED
@@ -16,7 +16,8 @@ export async function printLintResults(results: LintResult[], directories: strin
16
16
  const errors = result.issues.filter((i) => i.severity === 'error')
17
17
  if (errors.length > 0) hasError = true
18
18
 
19
- const dirLabel = result.directory === directories[0] && results.length === 1 ? result.directory : result.directory
19
+ const dirLabel =
20
+ result.directory === directories[0] && results.length === 1 ? result.directory : result.directory
20
21
  if (result.issues.length === 0) {
21
22
  tui.print(chalk.green('✓') + ' ' + dirLabel + ' ' + chalk.gray('— no issues'))
22
23
  } else {
@@ -35,9 +36,7 @@ export async function printLintResults(results: LintResult[], directories: strin
35
36
  const totalErrors = results.reduce((s, r) => s + r.issues.filter((i) => i.severity === 'error').length, 0)
36
37
  const totalWarnings = results.reduce((s, r) => s + r.issues.filter((i) => i.severity === 'warning').length, 0)
37
38
  if (totalErrors > 0 || totalWarnings > 0) {
38
- tui.gray(
39
- `Total: ${totalErrors} error(s), ${totalWarnings} warning(s) across ${results.length} problem(s)`,
40
- )
39
+ tui.gray(`Total: ${totalErrors} error(s), ${totalWarnings} warning(s) across ${results.length} problem(s)`)
41
40
  }
42
41
  }
43
42