kot 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/kot.rb +2 -1
- data/lib/kot/hill_climbing_estimator.rb +32 -35
- data/lib/kot/item4pl.rb +93 -0
- data/lib/kot/item_response_theory.rb +30 -22
- data/lib/kot/randomesque_selector.rb +3 -3
- data/lib/kot/test.rb +6 -4
- data/lib/kot/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4df58ff0365bfa26e18adba52d56d07b2ba6e1648a7672e30e36074086a59b86
|
4
|
+
data.tar.gz: 277e71849654efe4c68d6e0994195c03bfcc4758d5e768fcca3a363e56c54c01
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 782b27fe171ca5ed98fde2c0f83115bfc94e3272c25132779edff79f0955bff817238f53b4079bb122b563274054cd00d7399b967783abe2ad533ce8ea026642
|
7
|
+
data.tar.gz: 24ab8c8ae3af9e2bbf3bf02de19e2fbcdcbdcf552f840bb835c1bbe91cd3f4bbe27adfba0d26a0542f12c6e56245dac8e40d9171a71815d014bc8882b3015bab
|
data/lib/kot.rb
CHANGED
@@ -1,54 +1,53 @@
|
|
1
1
|
module Kot
|
2
|
-
|
3
2
|
class HillClimbingEstimator
|
4
3
|
|
5
|
-
|
6
|
-
|
4
|
+
# Estimates theta when all responses are true or all are false, based on Dodd, 1990.
|
7
5
|
# "The variable stepsize changed the 0 estimate by half the distance to the appropriate ... value in the item pool."
|
8
|
-
def dodd(est_theta:0.0, items:[], last_response:[])
|
6
|
+
def dodd(est_theta: 0.0, items: [], last_response: [])
|
9
7
|
max_b = items.map(&:b).max
|
10
8
|
min_b = items.map(&:b).min
|
11
9
|
|
12
10
|
last_response ? est_theta + ((max_b - est_theta) / 2.0) : est_theta - ((est_theta - min_b) / 2.0)
|
13
11
|
end
|
14
12
|
|
15
|
-
|
13
|
+
# Performs a single iteration of the hill climb, starting from one bound towards the other.
|
16
14
|
def estimate_iteration(best_theta, max_ll, lower_bound, upper_bound, responses, items)
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
intervals.each do |ii|
|
15
|
+
step_size = (upper_bound - lower_bound) / 10
|
16
|
+
|
17
|
+
case step_size <=> 0
|
18
|
+
when 1
|
19
|
+
intervals = (lower_bound..upper_bound).step(step_size).each
|
20
|
+
when -1
|
21
|
+
intervals = (upper_bound..lower_bound).step(step_size.abs).reverse_each
|
22
|
+
when 0
|
23
|
+
intervals = []
|
24
|
+
end
|
29
25
|
|
30
|
-
|
26
|
+
intervals.each do |ii|
|
31
27
|
|
32
|
-
|
33
|
-
max_ll = ll
|
28
|
+
ll = ItemResponseTheory.log_likelihood(ii, responses, items)
|
34
29
|
|
35
|
-
|
36
|
-
|
37
|
-
else
|
38
|
-
lower_bound = best_theta - step_size.abs
|
39
|
-
upper_bound = ii
|
40
|
-
break
|
41
|
-
end
|
30
|
+
if ll > max_ll
|
31
|
+
max_ll = ll
|
42
32
|
|
33
|
+
#TODO: precision-based early exit
|
34
|
+
best_theta = ii
|
35
|
+
else
|
36
|
+
lower_bound = best_theta - step_size.abs
|
37
|
+
upper_bound = ii
|
38
|
+
break
|
43
39
|
end
|
44
40
|
|
45
|
-
|
41
|
+
end
|
42
|
+
|
43
|
+
[best_theta, max_ll, lower_bound, upper_bound]
|
46
44
|
end
|
47
45
|
|
48
|
-
|
46
|
+
# Estimate theta using multiple iterations of a hill climb, falling back to {#dodd} if all responses are true or false.
|
47
|
+
def estimate(responses: [], items: [], all_items: [], est_theta: 0.0)
|
49
48
|
if responses.uniq.count == 1
|
50
|
-
raise ArgumentError
|
51
|
-
return dodd(est_theta:est_theta, items:all_items, last_response:responses.last)
|
49
|
+
raise ArgumentError, "Responses are all #{responses.first} but missing all_items argument" if all_items.empty?
|
50
|
+
return dodd(est_theta: est_theta, items: all_items, last_response: responses.last)
|
52
51
|
end
|
53
52
|
|
54
53
|
lower_bound = items.map(&:b).min
|
@@ -59,8 +58,6 @@ module Kot
|
|
59
58
|
best_theta = - Float::INFINITY
|
60
59
|
max_ll = - Float::INFINITY
|
61
60
|
|
62
|
-
old_best_theta = best_theta
|
63
|
-
|
64
61
|
10.times do
|
65
62
|
best_theta, max_ll, lower_bound, upper_bound =
|
66
63
|
estimate_iteration(best_theta, max_ll, lower_bound, upper_bound, responses, items)
|
@@ -68,9 +65,9 @@ module Kot
|
|
68
65
|
break if lower_bound == upper_bound
|
69
66
|
end
|
70
67
|
|
71
|
-
|
68
|
+
best_theta
|
72
69
|
end
|
73
70
|
|
74
71
|
end
|
75
72
|
|
76
|
-
end
|
73
|
+
end
|
data/lib/kot/item4pl.rb
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
module Kot
|
2
|
+
|
3
|
+
# An example of an Item class. You probably don't want to inherit from this,
|
4
|
+
# but instead to include {ItemResponseTheory} in a class of your own that
|
5
|
+
# provides {#a}, {#b}, {#c} and {#d}. See those attribute definitions for
|
6
|
+
# more information about what each parameter means in the context of
|
7
|
+
# correct/incorrect tests of ability.
|
8
|
+
#
|
9
|
+
# This class is useful for simulating different CAT setups and is used
|
10
|
+
# for some of the specification tests of this library.
|
11
|
+
class Item4PL
|
12
|
+
include Kot::ItemResponseTheory
|
13
|
+
|
14
|
+
attr_reader :a, :b, :c, :d
|
15
|
+
|
16
|
+
# @!attribute [r] a
|
17
|
+
# Discrimination ability of the item.
|
18
|
+
# This describe the maximum slope of the item's ICC, at the point given by {#b},
|
19
|
+
# and so how sharply the item distinguishes between those with ability below and above that point.
|
20
|
+
#
|
21
|
+
# An item with an _a_ of zero would have an entirely flat ICC (and so be completely useless),
|
22
|
+
# while an item with an infinitely high _a_ would perfectly discriminate such that anyone with an ability
|
23
|
+
# of *less* *than* _b_ would have _c_ probability of getting the answer right and anyone with an ability
|
24
|
+
# *greater* *than* _b_ would have _d_ probability of getting the answer right.
|
25
|
+
#
|
26
|
+
# Usually set to 1.0 for 1PL models.
|
27
|
+
# @return [Float] Discrimination
|
28
|
+
|
29
|
+
# @!attribute [r] b
|
30
|
+
# Difficulty of the item.
|
31
|
+
# This describes the midpoint of the item's ICC, where P(_b_)=0.5,
|
32
|
+
# and so the point at which half of those with a _theta_ equal to _b_ will answer
|
33
|
+
# the item correctly and half will answer it incorrectly.
|
34
|
+
#
|
35
|
+
# _b_ is the parameter required by any model, and across the bank of items usually
|
36
|
+
# will range over the distribution of test-takers' ability.
|
37
|
+
# (Often the distribution of ability is conceived as N(0,1).)
|
38
|
+
#
|
39
|
+
# @return [Float] Difficulty
|
40
|
+
|
41
|
+
# @!attribute [r] c
|
42
|
+
# Likelihood of guesing the item.
|
43
|
+
# This describes the lower asymptote of the item's ICC,
|
44
|
+
# and so the lowest probability of answering correctly regardless of ability.
|
45
|
+
#
|
46
|
+
# For example, a multiple choice test item with 5 possible answers,
|
47
|
+
# one of which is correct, might have a _c_ of 1/5 or 0.2 .
|
48
|
+
#
|
49
|
+
# Usually set to 0.0 for 1-2PL models.
|
50
|
+
# @return [Float] Guessing
|
51
|
+
|
52
|
+
# @!attribute [r] d
|
53
|
+
# Maximum likelihood of answering correctly.
|
54
|
+
# This describes the upper asymptote of the item's ICC,
|
55
|
+
# and so the highest probability of answering correctly regardless of ability.
|
56
|
+
#
|
57
|
+
# This parameter is more common in contexts outside of testing ability
|
58
|
+
# such as cognitive and clinical measures.
|
59
|
+
#
|
60
|
+
# Usually set to 1.0 for 1-3PL models.
|
61
|
+
# @return [Float] Insurmountable difficulty
|
62
|
+
|
63
|
+
def self.[](*arr)
|
64
|
+
arr.map { |a| Item4PL.new(a) }
|
65
|
+
end
|
66
|
+
|
67
|
+
# @see https://stackoverflow.com/questions/5825680/code-to-generate-gaussian-normally-distributed-random-numbers-in-ruby
|
68
|
+
def self.gaussian(mean = 0.0, stddev = 1.0, rand = lambda{ Kernel.rand } )
|
69
|
+
theta = 2 * Math::PI * rand.call
|
70
|
+
rho = Math.sqrt(-2 * Math.log(1 - rand.call))
|
71
|
+
scale = stddev * rho
|
72
|
+
mean + scale * Math.cos(theta)
|
73
|
+
end
|
74
|
+
|
75
|
+
# @return [Item4PL] a 1PL Item4PL with a {#b} chosen randomly from N(0,1)
|
76
|
+
# @return [Array] an array of 1PL Item4PLs with {#b}s chosen randomly from N(0,1)
|
77
|
+
def self.generate(n = nil)
|
78
|
+
return Item4PL.new(b:gaussian()) if n.nil?
|
79
|
+
Array.new(n) { generate }
|
80
|
+
end
|
81
|
+
|
82
|
+
def initialize(a: 1.0, b: 0.0, c: 0.0, d: 1.0)
|
83
|
+
@a = a
|
84
|
+
@b = b
|
85
|
+
@c = c
|
86
|
+
@d = d
|
87
|
+
end
|
88
|
+
|
89
|
+
def to_s
|
90
|
+
"<Item4PL a:#{a} b:#{b} c:#{c} d#{d} >"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -1,53 +1,61 @@
|
|
1
1
|
module Kot
|
2
2
|
|
3
|
-
#
|
3
|
+
# Include this module into a class to give various IRT statistics for individual items.
|
4
|
+
# Including classes are expected to respond to #a, #b, #c and #d ; see the example {Item4PL} class for more information.
|
5
|
+
#
|
6
|
+
# Class methods for this module provide IRT statistics for a set of items,
|
7
|
+
# given an individual's estimated theta (and sometimes their responses to those items).
|
4
8
|
module ItemResponseTheory
|
5
9
|
|
6
|
-
#
|
7
|
-
#
|
8
|
-
#
|
9
|
-
|
10
|
+
# @param est_theta [Float] an estimate of an individual's ability
|
11
|
+
# @param responses [Array<TrueClass, FalseClass>] the responses given by an individual to _items_
|
12
|
+
# @param items [Array<ItemResponseTheory>] items that an individual has responded to
|
13
|
+
# @return [Float]
|
10
14
|
def self.log_likelihood(est_theta, responses, items)
|
11
|
-
ps = items.map {|i| i.icc(est_theta) }
|
12
|
-
ls = ps.each_with_index.map {|e,i| responses[i] ? Math.log(e) : Math.log(1.0 - e)}
|
15
|
+
ps = items.map { |i| i.icc(est_theta) }
|
16
|
+
ls = ps.each_with_index.map { |e, i| responses[i] ? Math.log(e) : Math.log(1.0 - e) }
|
17
|
+
# TODO: Polychotomous
|
13
18
|
ls.inject(:+)
|
14
19
|
end
|
15
20
|
|
21
|
+
# @param est_theta [Float] an estimate of an individual's ability
|
22
|
+
# @param items [Array<ItemResponseTheory>] items that an individual has responded to
|
23
|
+
# @return [Float]
|
16
24
|
def self.test_info(est_theta, items)
|
17
|
-
items.map {|i| i.inf(est_theta) }.inject(:+)
|
25
|
+
items.map { |i| i.inf(est_theta) }.inject(:+)
|
18
26
|
end
|
19
27
|
|
28
|
+
# @param est_theta [Float] an estimate of an individual's ability
|
29
|
+
# @param items [Array<ItemResponseTheory>] items that an individual has responded to
|
30
|
+
# @return [Float]
|
20
31
|
def self.var(est_theta, items)
|
21
|
-
1.0/test_info(est_theta, items)
|
32
|
+
1.0 / test_info(est_theta, items)
|
22
33
|
end
|
23
34
|
|
35
|
+
# @param est_theta [Float] an estimate of an individual's ability
|
36
|
+
# @param items [Array<ItemResponseTheory>] items that an individual has responded to
|
37
|
+
# @return [Float] standard error of estimation
|
24
38
|
def self.see(est_theta, items)
|
25
39
|
Math.sqrt(var(est_theta, items))
|
26
40
|
end
|
27
41
|
|
28
|
-
|
29
|
-
#
|
30
|
-
# Methods intended for inclusion into Item-classes
|
31
|
-
#
|
32
|
-
|
33
|
-
def icc_component(theta)
|
34
|
-
Math.exp(-a * (theta - b))
|
35
|
-
end
|
36
|
-
|
37
42
|
# Item characteristic curve
|
43
|
+
# @return [Float] the probability of someone with ability _theta_ of answering this item correctly
|
38
44
|
def icc(theta)
|
39
|
-
|
45
|
+
icc_component = Math.exp(-a * (theta - b))
|
46
|
+
c + ((d - c) / (1.0 + icc_component))
|
40
47
|
end
|
41
48
|
|
42
|
-
#
|
49
|
+
# Item information function
|
50
|
+
# @return [Float] a measure of the information that would be provided by a response to this item given a prior _theta_
|
43
51
|
def inf(theta)
|
44
52
|
vp = icc(theta)
|
45
53
|
|
46
54
|
top = (a ** 2) * ((vp - c) ** 2) * ((d - vp) ** 2)
|
47
55
|
bottom = ((d - c) ** 2) * vp * (1.0 - vp)
|
48
56
|
|
49
|
-
top/bottom
|
57
|
+
top / bottom
|
50
58
|
end
|
51
59
|
|
52
60
|
end
|
53
|
-
end
|
61
|
+
end
|
data/lib/kot/test.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
module Kot
|
2
2
|
|
3
|
-
|
4
3
|
class Test
|
5
4
|
|
6
5
|
attr_reader :est_theta
|
@@ -15,21 +14,24 @@ module Kot
|
|
15
14
|
@asked_items = []
|
16
15
|
end
|
17
16
|
|
18
|
-
|
17
|
+
# Get the standard error of estimation for the test so far.
|
19
18
|
def see
|
20
19
|
return Float::INFINITY if @asked_items.empty?
|
21
20
|
ItemResponseTheory.see(@est_theta, @asked_items)
|
22
21
|
end
|
23
22
|
|
23
|
+
# Update the estimated theta for the test so far.
|
24
24
|
def update_est_theta
|
25
|
-
@est_theta = @estimator.estimate(est_theta: @est_theta, responses
|
25
|
+
@est_theta = @estimator.estimate(est_theta: @est_theta, responses: @responses, items: @asked_items, all_items: @item_bank)
|
26
26
|
end
|
27
27
|
|
28
|
+
# Ask the selector for a new item from the item bank.
|
28
29
|
def next_item
|
29
30
|
possible_items = @item_bank - @asked_items
|
30
31
|
@selector.select(@est_theta, possible_items)
|
31
32
|
end
|
32
33
|
|
34
|
+
# Add a response for a given item.
|
33
35
|
def respond(response, item)
|
34
36
|
@responses << response
|
35
37
|
@asked_items << item
|
@@ -38,4 +40,4 @@ module Kot
|
|
38
40
|
|
39
41
|
end
|
40
42
|
|
41
|
-
end
|
43
|
+
end
|
data/lib/kot/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kot
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Watkins
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-10-03 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: " Kot is a basic toolkit for getting started with computerised adaptive
|
14
14
|
testing (CAT). It includes a module to calculate item response theory (IRT) statistics
|
@@ -22,6 +22,7 @@ extra_rdoc_files: []
|
|
22
22
|
files:
|
23
23
|
- lib/kot.rb
|
24
24
|
- lib/kot/hill_climbing_estimator.rb
|
25
|
+
- lib/kot/item4pl.rb
|
25
26
|
- lib/kot/item_response_theory.rb
|
26
27
|
- lib/kot/randomesque_selector.rb
|
27
28
|
- lib/kot/test.rb
|